[Multiple Datasource] Support Amazon OpenSearch Serverless (#3957) (#4066)

* [Multiple Datasource]Support Amazon OpenSearch Serverless in SigV4
* remove experimental text in yml
* Refactor create data source form for authentication

Signed-off-by: Su <szhongna@amazon.com>
(cherry picked from commit e737790)

Author: zhongnansu

config/opensearch_dashboards.yml

@@ -229,8 +229,7 @@
 # functionality in Visualization.
 # vis_builder.enabled: false
 
-# Set the value of this setting to true to enable the experimental multiple data source
-# support feature. Use with caution.
+# Set the value of this setting to true to enable multiple data source feature.
 #data_source.enabled: false
 # Set the value of these settings to customize crypto materials to encryption saved credentials
 # in data sources.

package.json

@@ -139,7 +139,7 @@
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
     "@opensearch-project/opensearch": "^1.1.0",
-    "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.1.0",
+    "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.2.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",
@@ -172,7 +172,7 @@
     "dns-sync": "^0.2.1",
     "elastic-apm-node": "^3.7.0",
     "elasticsearch": "^16.7.0",
-    "http-aws-es": "6.0.0",
+    "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1",
     "execa": "^4.0.2",
     "expiry-js": "0.1.7",
     "fast-deep-equal": "^3.1.1",

src/plugins/data_source/common/data_sources/types.ts

@@ -25,6 +25,7 @@ export interface SigV4Content extends SavedObjectAttributes {
   accessKey: string;
   secretKey: string;
   region: string;
+  service?: SigV4ServiceName;
 }
 
 export interface UsernamePasswordTypedContent extends SavedObjectAttributes {
@@ -37,3 +38,8 @@ export enum AuthType {
   UsernamePasswordType = 'username_password',
   SigV4 = 'sigv4',
 }
+
+export enum SigV4ServiceName {
+  OpenSearch = 'es',
+  OpenSearchServerless = 'aoss',
+}

src/plugins/data_source/opensearch_dashboards.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": [],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["common/data_sources"]
 }

src/plugins/data_source/server/client/configure_client.test.ts

@@ -167,6 +167,30 @@ describe('configureClient', () => {
     expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(2);
   });
 
+  test('configure client with auth.type == sigv4, service == aoss, should successfully call new Client()', async () => {
+    savedObjectsMock.get.mockReset().mockResolvedValueOnce({
+      id: DATA_SOURCE_ID,
+      type: DATA_SOURCE_SAVED_OBJECT_TYPE,
+      attributes: {
+        ...dataSourceAttr,
+        auth: {
+          type: AuthType.SigV4,
+          credentials: { ...sigV4AuthContent, service: 'aoss' },
+        },
+      },
+      references: [],
+    });
+
+    jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
+      decryptedText: 'accessKey',
+      encryptionContext: { endpoint: 'http://localhost' },
+    });
+
+    await configureClient(dataSourceClientParams, clientPoolSetup, config, logger);
+
+    expect(ClientMock).toHaveBeenCalledTimes(1);
+  });
+
   test('configure test client for non-exist datasource should not call saved object api, nor decode any credential', async () => {
     const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
       decryptedText: 'password',

src/plugins/data_source/server/client/configure_client.ts

@@ -160,7 +160,7 @@ const getBasicAuthClient = (
 };
 
 const getAWSClient = (credential: SigV4Content, clientOptions: ClientOptions): Client => {
-  const { accessKey, secretKey, region } = credential;
+  const { accessKey, secretKey, region, service } = credential;
 
   const credentialProvider = (): Promise<Credentials> => {
     return new Promise((resolve) => {
@@ -172,6 +172,7 @@ const getAWSClient = (credential: SigV4Content, clientOptions: ClientOptions): C
     ...AwsSigv4Signer({
       region,
       getCredentials: credentialProvider,
+      service,
     }),
     ...clientOptions,
   });

src/plugins/data_source/server/client/configure_client_utils.ts

@@ -91,7 +91,7 @@ export const getAWSCredential = async (
   cryptography: CryptographyServiceSetup
 ): Promise<SigV4Content> => {
   const { endpoint } = dataSource;
-  const { accessKey, secretKey, region } = dataSource.auth.credentials! as SigV4Content;
+  const { accessKey, secretKey, region, service } = dataSource.auth.credentials! as SigV4Content;
 
   const {
     decryptedText: accessKeyText,
@@ -122,6 +122,7 @@ export const getAWSCredential = async (
     region,
     accessKey: accessKeyText,
     secretKey: secretKeyText,
+    service,
   };
 
   return credential;

src/plugins/data_source/server/legacy/configure_legacy_client.test.ts

@@ -6,7 +6,7 @@
 import { SavedObjectsClientContract } from '../../../../core/server';
 import { loggingSystemMock, savedObjectsClientMock } from '../../../../core/server/mocks';
 import { DATA_SOURCE_SAVED_OBJECT_TYPE } from '../../common';
-import { AuthType, DataSourceAttributes } from '../../common/data_sources';
+import { AuthType, DataSourceAttributes, SigV4Content } from '../../common/data_sources';
 import { DataSourcePluginConfigType } from '../../config';
 import { cryptographyServiceSetupMock } from '../cryptography_service.mocks';
 import { CryptographyServiceSetup } from '../cryptography_service';
@@ -27,6 +27,7 @@ describe('configureLegacyClient', () => {
   let clientPoolSetup: OpenSearchClientPoolSetup;
   let configOptions: ConfigOptions;
   let dataSourceAttr: DataSourceAttributes;
+  let sigV4AuthContent: SigV4Content;
 
   let mockOpenSearchClientInstance: {
     close: jest.Mock;
@@ -71,6 +72,12 @@ describe('configureLegacyClient', () => {
       },
     } as DataSourceAttributes;
 
+    sigV4AuthContent = {
+      region: 'us-east-1',
+      accessKey: 'accessKey',
+      secretKey: 'secretKey',
+    };
+
     clientPoolSetup = {
       getClientFromPool: jest.fn(),
       addClientToPool: jest.fn(),
@@ -157,6 +164,42 @@ describe('configureLegacyClient', () => {
     expect(mockResult).toBeDefined();
   });
 
+  test('configure client with auth.type == sigv4 and service param, should call new Client() with service param', async () => {
+    savedObjectsMock.get.mockReset().mockResolvedValueOnce({
+      id: DATA_SOURCE_ID,
+      type: DATA_SOURCE_SAVED_OBJECT_TYPE,
+      attributes: {
+        ...dataSourceAttr,
+        auth: {
+          type: AuthType.SigV4,
+          credentials: { ...sigV4AuthContent, service: 'aoss' },
+        },
+      },
+      references: [],
+    });
+
+    parseClientOptionsMock.mockReturnValue(configOptions);
+
+    jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
+      decryptedText: 'accessKey',
+      encryptionContext: { endpoint: 'http://localhost' },
+    });
+
+    await configureLegacyClient(
+      dataSourceClientParams,
+      callApiParams,
+      clientPoolSetup,
+      config,
+      logger
+    );
+
+    expect(parseClientOptionsMock).toHaveBeenCalled();
+    expect(ClientMock).toHaveBeenCalledTimes(1);
+    expect(ClientMock).toHaveBeenCalledWith(expect.objectContaining({ service: 'aoss' }));
+
+    expect(savedObjectsMock.get).toHaveBeenCalledTimes(1);
+  });
+
   test('configure client with auth.type == username_password and password contaminated', async () => {
     const decodeAndDecryptSpy = jest
       .spyOn(cryptographyMock, 'decodeAndDecrypt')

src/plugins/data_source/server/legacy/configure_legacy_client.ts

@@ -5,10 +5,9 @@
 
 import { Client } from '@opensearch-project/opensearch-next';
 import { Client as LegacyClient, ConfigOptions } from 'elasticsearch';
-import { Credentials } from 'aws-sdk';
+import { Credentials, Config } from 'aws-sdk';
 import { get } from 'lodash';
 import HttpAmazonESConnector from 'http-aws-es';
-import { Config } from 'aws-sdk';
 import {
   Headers,
   LegacyAPICaller,
@@ -27,7 +26,7 @@ import { CryptographyServiceSetup } from '../cryptography_service';
 import { DataSourceClientParams, LegacyClientCallAPIParams } from '../types';
 import { OpenSearchClientPoolSetup } from '../client';
 import { parseClientOptions } from './client_config';
-import { createDataSourceError, DataSourceError } from '../lib/error';
+import { createDataSourceError } from '../lib/error';
 import {
   getRootClient,
   getAWSCredential,
@@ -195,13 +194,14 @@ const getBasicAuthClient = async (
 };
 
 const getAWSClient = (credential: SigV4Content, clientOptions: ConfigOptions): LegacyClient => {
-  const { accessKey, secretKey, region } = credential;
+  const { accessKey, secretKey, region, service } = credential;
   const client = new LegacyClient({
     connectionClass: HttpAmazonESConnector,
     awsConfig: new Config({
       region,
       credentials: new Credentials({ accessKeyId: accessKey, secretAccessKey: secretKey }),
     }),
+    service,
     ...clientOptions,
   });
   return client;

src/plugins/data_source/server/routes/data_source_connection_validator.ts

@@ -5,13 +5,19 @@
 
 import { OpenSearchClient } from 'opensearch-dashboards/server';
 import { createDataSourceError } from '../lib/error';
-
+import { SigV4ServiceName } from '../../common/data_sources';
 export class DataSourceConnectionValidator {
-  constructor(private readonly callDataCluster: OpenSearchClient) {}
+  constructor(
+    private readonly callDataCluster: OpenSearchClient,
+    private readonly dataSourceAttr: any
+  ) {}
 
   async validate() {
     try {
-      return await this.callDataCluster.info<OpenSearchClient>();
+      // Amazon OpenSearch Serverless does not support .info() API
+      if (this.dataSourceAttr.auth?.credentials?.service === SigV4ServiceName.OpenSearchServerless)
+        return await this.callDataCluster.cat.indices();
+      return await this.callDataCluster.info();
     } catch (e) {
       throw createDataSourceError(e);
     }

src/plugins/data_source/server/routes/test_connection.ts

@@ -5,7 +5,7 @@
 
 import { schema } from '@osd/config-schema';
 import { IRouter, OpenSearchClient } from 'opensearch-dashboards/server';
-import { AuthType, DataSourceAttributes } from '../../common/data_sources';
+import { AuthType, DataSourceAttributes, SigV4ServiceName } from '../../common/data_sources';
 import { DataSourceConnectionValidator } from './data_source_connection_validator';
 import { DataSourceServiceSetup } from '../data_source_service';
 import { CryptographyServiceSetup } from '../cryptography_service';
@@ -40,6 +40,10 @@ export const registerTestConnectionRoute = (
                       region: schema.string(),
                       accessKey: schema.string(),
                       secretKey: schema.string(),
+                      service: schema.oneOf([
+                        schema.literal(SigV4ServiceName.OpenSearch),
+                        schema.literal(SigV4ServiceName.OpenSearchServerless),
+                      ]),
                     }),
                   ])
                 ),
@@ -61,9 +65,13 @@ export const registerTestConnectionRoute = (
             testClientDataSourceAttr: dataSourceAttr as DataSourceAttributes,
           }
         );
-        const dsValidator = new DataSourceConnectionValidator(dataSourceClient);
 
-        await dsValidator.validate();
+        const dataSourceValidator = new DataSourceConnectionValidator(
+          dataSourceClient,
+          dataSourceAttr
+        );
+
+        await dataSourceValidator.validate();
 
         return response.ok({
           body: {

src/plugins/data_source/server/saved_objects/data_source_saved_objects_client_wrapper.ts

@@ -301,7 +301,7 @@ export class DataSourceSavedObjectsClientWrapper {
           );
         }
 
-        const { accessKey, secretKey, region } = credentials as SigV4Content;
+        const { accessKey, secretKey, region, service } = credentials as SigV4Content;
 
         if (!accessKey) {
           throw SavedObjectsErrorHelpers.createBadRequestError(
@@ -320,6 +320,12 @@ export class DataSourceSavedObjectsClientWrapper {
             '"auth.credentials.region" attribute is required'
           );
         }
+
+        if (!service) {
+          throw SavedObjectsErrorHelpers.createBadRequestError(
+            '"auth.credentials.service" attribute is required'
+          );
+        }
         break;
       default:
         throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid auth type: '${type}'`);
@@ -457,7 +463,7 @@ export class DataSourceSavedObjectsClientWrapper {
 
   private async encryptSigV4Credential<T = unknown>(auth: T, encryptionContext: EncryptionContext) {
     const {
-      credentials: { accessKey, secretKey, region },
+      credentials: { accessKey, secretKey, region, service },
     } = auth;
 
     return {
@@ -466,6 +472,7 @@ export class DataSourceSavedObjectsClientWrapper {
         region,
         accessKey: await this.cryptography.encryptAndEncode(accessKey, encryptionContext),
         secretKey: await this.cryptography.encryptAndEncode(secretKey, encryptionContext),
+        service,
       },
     };
   }

src/plugins/data_source_management/public/components/create_data_source_wizard/components/create_form/create_data_source_form.test.tsx

@@ -27,6 +27,7 @@ describe('Datasource Management: Create Datasource form', () => {
   let component: ReactWrapper<any, Readonly<{}>, React.Component<{}, {}, any>>;
   const mockSubmitHandler = jest.fn();
   const mockTestConnectionHandler = jest.fn();
+  const mockCancelHandler = jest.fn();
 
   const getFields = (comp: ReactWrapper<any, Readonly<{}>, React.Component<{}, {}, any>>) => {
     return {
@@ -65,6 +66,7 @@ describe('Datasource Management: Create Datasource form', () => {
         <CreateDataSourceForm
           handleTestConnection={mockTestConnectionHandler}
           handleSubmit={mockSubmitHandler}
+          handleCancel={mockCancelHandler}
           existingDatasourceNamesList={['dup20']}
         />
       ),