[7.17] Debugging with apm - fixes and tutorial (#127892) (#129600)

* Debugging with apm - fixes and tutorial (#127892)

* fixes + tutorial

* cors config

* omit secretToken so its not sent to FE
Add config tests

* lint

* empty

* swallow errors when parsing configs

* read config test adjustment

* apm docs review

* new line

* doc review

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit c97bfc8)

# Conflicts:
#	packages/kbn-apm-config-loader/src/config.test.ts
#	packages/kbn-apm-config-loader/src/config.ts
#	src/core/server/http_resources/get_apm_config.ts

* fix merge

Author: Liza Katz

dev_docs/tutorials/apm_ui.png

@@ -61,3 +61,49 @@ logging:
     - name: elasticsearch.query
       level: debug
 ```
+
+## Debugging Kibana with APM
+
+Kibana is integrated with APM's node and RUM agents. 
+To learn more about how APM works and what it reports, refer to the [documentation](https://www.elastic.co/guide/en/apm/guide/current/index.html).
+
+We currently track the following types of transactions from Kibana: 
+
+Frontend (APM RUM): 
+ * `http-request`- tracks all outgoing API requests
+ * `page-load` - tracks the inidial loading time of kibana
+ * `app-change` - tracks application changes
+
+Backend (APM Node): 
+ * `request` - tracks all incoming API requests
+ * `kibana-platform` - tracks server initiation phases (preboot, setup and start)
+ * `task-manager` - tracks the operation of the task manager, including claiming pending tasks and marking them as running
+ * `task-run` - tracks the execution of individual tasks
+
+### Enabling APM on a local environment
+
+In some cases, it is beneficial to enable APM on a local development environment to get an initial undesrtanding of a feature's performance during manual or automatic tests.
+
+ 1. Create a secondary monitoring deployment to collect APM data. The easiest option is to use [Elastic Cloud](https://cloud.elastic.co/deployments) to create a new deployment.
+ 2. Open Kibana, go to `Integrations` and enable the Elastic APM integration.
+ 3. Scroll down and copy the server URL and secret token. You may also find them in your cloud console under APM & Fleet.
+ 4. Create or open `config\kibana.dev.yml` on your local development environment.
+ 5. Add the following settings: 
+    ```
+    elastic.apm.active: true
+    elastic.apm.serverUrl: <serverUrl>
+    elastic.apm.secretToken: <secretToken>
+    ```
+ 6. Once you run kibana and start using it, two new services (kibana, kibana-frontend) should appear under the APM UI on the APM deployment.
+  ![APM UI](./apm_ui.png)
+
+### Enabling APM via environment variables
+
+It is possible to enable APM via environment variables as well.
+They take precedence over any values defined in `kibana.yml` or `kibana.dev.yml`
+
+Set the following environment variables to enable APM: 
+
+ * ELASTIC_APM_ACTIVE
+ * ELASTIC_APM_SERVER_URL
+ * ELASTIC_APM_SECRET_TOKEN

dev_docs/tutorials/debugging.mdx

@@ -17,13 +17,6 @@ export const packageMock = {
 };
 jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true });
 
-export const devConfigMock = {
-  raw: {} as any,
-};
-jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, {
-  virtual: true,
-});
-
 export const gitRevExecMock = jest.fn();
 jest.doMock('child_process', () => ({
   ...childProcessModule,
@@ -48,7 +41,6 @@ jest.doMock('fs', () => ({
 
 export const resetAllMocks = () => {
   packageMock.raw = {};
-  devConfigMock.raw = {};
   gitRevExecMock.mockReset();
   readUuidFileMock.mockReset();
   jest.resetModules();

packages/kbn-apm-config-loader/src/config.test.mocks.ts

@@ -5,23 +5,21 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import type { Labels } from 'elastic-apm-node';
+import type { AgentConfigOptions, Labels } from 'elastic-apm-node';
 import {
   packageMock,
   mockedRootDir,
   gitRevExecMock,
-  devConfigMock,
   readUuidFileMock,
   resetAllMocks,
 } from './config.test.mocks';
 
-import { ApmConfiguration } from './config';
+import { ApmConfiguration, CENTRALIZED_SERVICE_BASE_CONFIG } from './config';
 
 describe('ApmConfiguration', () => {
   beforeEach(() => {
     // start with an empty env to avoid CI from spoiling snapshots, env is unique for each jest file
     process.env = {};
-
     packageMock.raw = {
       version: '8.0.0',
       build: {
@@ -148,82 +146,57 @@ describe('ApmConfiguration', () => {
     );
   });
 
-  it('loads the configuration from the dev config is present', () => {
-    devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
-    };
-    const config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: true,
-        serverUrl: 'https://dev-url.co',
-      })
-    );
-  });
-
-  it('does not load the configuration from the dev config in distributable', () => {
-    devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
-    };
-    const config = new ApmConfiguration(mockedRootDir, {}, true);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: false,
-      })
-    );
-  });
-
-  it('overwrites the standard config file with the dev config', () => {
-    const kibanaConfig = {
-      elastic: {
-        apm: {
-          active: true,
-          serverUrl: 'https://url',
-          secretToken: 'secret',
-        },
-      },
-    };
-    devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
-    };
-    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: true,
-        serverUrl: 'https://dev-url.co',
-        secretToken: 'secret',
-      })
-    );
-  });
-
-  it('correctly sets environment by reading env vars', () => {
-    delete process.env.ELASTIC_APM_ENVIRONMENT;
-    delete process.env.NODE_ENV;
-
-    let config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'development',
-      })
-    );
-
-    process.env.NODE_ENV = 'production';
-    config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'production',
-      })
-    );
-
-    process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
-    config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'ci',
-      })
-    );
+  describe('env vars', () => {
+    beforeEach(() => {
+      delete process.env.ELASTIC_APM_ENVIRONMENT;
+      delete process.env.ELASTIC_APM_SECRET_TOKEN;
+      delete process.env.ELASTIC_APM_SERVER_URL;
+      delete process.env.NODE_ENV;
+    });
+
+    it('correctly sets environment by reading env vars', () => {
+      let config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'development',
+        })
+      );
+
+      process.env.NODE_ENV = 'production';
+      config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'production',
+        })
+      );
+
+      process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
+      config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'ci',
+        })
+      );
+    });
+
+    it('uses default config if serverUrl is not set', () => {
+      process.env.ELASTIC_APM_SECRET_TOKEN = 'banana';
+      const config = new ApmConfiguration(mockedRootDir, {}, false);
+      const serverConfig = config.getConfig('serviceName');
+      expect(serverConfig).toHaveProperty(
+        'secretToken',
+        (CENTRALIZED_SERVICE_BASE_CONFIG as AgentConfigOptions).secretToken
+      );
+      expect(serverConfig).toHaveProperty('serverUrl', CENTRALIZED_SERVICE_BASE_CONFIG.serverUrl);
+    });
+
+    it('uses env vars config if serverUrl is set', () => {
+      process.env.ELASTIC_APM_SECRET_TOKEN = 'banana';
+      process.env.ELASTIC_APM_SERVER_URL = 'http://banana.com/';
+      const config = new ApmConfiguration(mockedRootDir, {}, false);
+      const serverConfig = config.getConfig('serviceName');
+      expect(serverConfig).toHaveProperty('secretToken', process.env.ELASTIC_APM_SECRET_TOKEN);
+      expect(serverConfig).toHaveProperty('serverUrl', process.env.ELASTIC_APM_SERVER_URL);
+    });
   });
 });

packages/kbn-apm-config-loader/src/config.test.ts

@@ -23,7 +23,7 @@ const DEFAULT_CONFIG: AgentConfigOptions = {
   globalLabels: {},
 };
 
-const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = {
+export const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = {
   serverUrl: 'https://kibana-cloud-apm.apm.us-east-1.aws.found.io',
 
   // The secretToken below is intended to be hardcoded in this file even though
@@ -74,6 +74,8 @@ export class ApmConfiguration {
 
   private getBaseConfig() {
     if (!this.baseConfig) {
+      const configFromSources = this.getConfigFromAllSources();
+
       this.baseConfig = merge(
         {
           serviceVersion: this.kibanaVersion,
@@ -82,9 +84,7 @@ export class ApmConfiguration {
         this.getUuidConfig(),
         this.getGitConfig(),
         this.getCiConfig(),
-        this.getConfigFromKibanaConfig(),
-        this.getDevConfig(),
-        this.getConfigFromEnv()
+        configFromSources
       );
 
       /**
@@ -129,6 +129,10 @@ export class ApmConfiguration {
       config.serverUrl = process.env.ELASTIC_APM_SERVER_URL;
     }
 
+    if (process.env.ELASTIC_APM_SECRET_TOKEN) {
+      config.secretToken = process.env.ELASTIC_APM_SECRET_TOKEN;
+    }
+
     if (process.env.ELASTIC_APM_GLOBAL_LABELS) {
       config.globalLabels = Object.fromEntries(
         process.env.ELASTIC_APM_GLOBAL_LABELS.split(',').map((p) => {
@@ -149,23 +153,6 @@ export class ApmConfiguration {
     return this.rawKibanaConfig?.elastic?.apm ?? {};
   }
 
-  /**
-   * Get the configuration from the apm.dev.js file, supersedes config
-   * from the --config file, disabled when running the distributable
-   */
-  private getDevConfig(): AgentConfigOptions {
-    if (this.isDistributable) {
-      return {};
-    }
-
-    try {
-      const apmDevConfigPath = join(this.rootDir, 'config', 'apm.dev.js');
-      return require(apmDevConfigPath);
-    } catch (e) {
-      return {};
-    }
-  }
-
   /**
    * Determine the Kibana UUID, initialized the value of `globalLabels.kibana_uuid`
    * when the UUID can be determined.
@@ -254,4 +241,11 @@ export class ApmConfiguration {
       return {};
     }
   }
+
+  /**
+   * Reads APM configuration from different sources and merges them together.
+   */
+  private getConfigFromAllSources(): AgentConfigOptions {
+    return merge({}, this.getConfigFromKibanaConfig(), this.getConfigFromEnv());
+  }
 }

packages/kbn-apm-config-loader/src/config.ts

@@ -23,6 +23,10 @@ describe('getConfigurationFilePaths', () => {
   });
 
   it('fallbacks to `getConfigPath` value', () => {
-    expect(getConfigurationFilePaths([])).toEqual([getConfigPath()]);
+    const path = getConfigPath();
+    expect(getConfigurationFilePaths([])).toEqual([
+      path,
+      path.replace('kibana.yml', 'kibana.dev.yml'),
+    ]);
   });
 });

packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap

@@ -22,5 +22,9 @@ export const getConfigurationFilePaths = (argv: string[]): string[] => {
   if (rawPaths.length) {
     return rawPaths.map((path) => resolve(process.cwd(), path));
   }
-  return [getConfigPath()];
+
+  const configPath = getConfigPath();
+
+  // Pick up settings from dev.yml as well
+  return [configPath, configPath.replace('kibana.yml', 'kibana.dev.yml')];
 };

packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts

@@ -29,6 +29,12 @@ test('reads and merges multiple yaml files from file system and parses to json',
   expect(config).toMatchSnapshot();
 });
 
+test('reads yaml files from file system and parses to json, even if one is missing', () => {
+  const config = getConfigFromFiles([fixtureFile('one.yml'), fixtureFile('boo.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
 test('should inject an environment variable value when setting a value with ${ENV_VAR}', () => {
   process.env.KBN_ENV_VAR1 = 'val1';
   process.env.KBN_ENV_VAR2 = 'val2';
@@ -61,8 +67,9 @@ describe('different cwd()', () => {
     expect(config).toMatchSnapshot();
   });
 
-  test('fails to load relative paths, not found because of the cwd', () => {
+  test('ignores errors loading relative paths', () => {
     const relativePath = relative(resolve(__dirname, '..', '..'), fixtureFile('one.yml'));
-    expect(() => getConfigFromFiles([relativePath])).toThrowError(/ENOENT/);
+    const config = getConfigFromFiles([relativePath]);
+    expect(config).toStrictEqual({});
   });
 });