Add the @kbn/apm-config-loader package (#77855) (#78585)

* first shot of the apm configuration loader

* revert changes to kibana config

* remove test files for now

* remove `?.` usages

* use lazy config init to avoid crashing integration test runner

* loader improvements

* add config value override via cli args

* add tests for utils package

* add prod/dev config handling + loader tests

* add tests for config

* address josh's comments

* nit on doc

Author: pgayvallet

package.json

@@ -124,6 +124,7 @@
     "@hapi/good-squeeze": "5.2.1",
     "@hapi/wreck": "^15.0.2",
     "@kbn/analytics": "1.0.0",
+    "@kbn/apm-config-loader": "1.0.0",
     "@kbn/babel-preset": "1.0.0",
     "@kbn/config": "1.0.0",
     "@kbn/config-schema": "1.0.0",

packages/kbn-apm-config-loader/README.md

@@ -0,0 +1,13 @@
+# @kbn/apm-config-loader
+
+Configuration loader for the APM instrumentation script.
+
+This module is only meant to be used by the APM instrumentation script (`src/apm.js`)
+to load the required configuration options from the `kibana.yaml` configuration file with
+default values.
+
+### Why not just use @kbn-config?
+
+`@kbn/config` is the recommended way to load and read the kibana configuration file,
+however in the specific case of APM, we want to only need the minimal dependencies
+before loading `elastic-apm-node` to avoid losing instrumentation on the already loaded modules.

packages/kbn-apm-config-loader/__fixtures__/config.yml

@@ -0,0 +1,11 @@
+pid:
+  enabled: true
+  file: '/var/run/kibana.pid'
+  obj: { val: 3 }
+  arr: [1]
+  empty_obj: {}
+  empty_arr: []
+obj: { val: 3 }
+arr: [1, 2]
+empty_obj: {}
+empty_arr: []

packages/kbn-apm-config-loader/__fixtures__/config_flat.yml

@@ -0,0 +1,6 @@
+pid.enabled: true
+pid.file: '/var/run/kibana.pid'
+pid.obj: { val: 3 }
+pid.arr: [1, 2]
+pid.empty_obj: {}
+pid.empty_arr: []

packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml

@@ -0,0 +1,5 @@
+foo: 1
+bar: "pre-${KBN_ENV_VAR1}-mid-${KBN_ENV_VAR2}-post"
+
+elasticsearch:
+  requestHeadersWhitelist: ["${KBN_ENV_VAR1}", "${KBN_ENV_VAR2}"]

packages/kbn-apm-config-loader/__fixtures__/one.yml

@@ -0,0 +1,9 @@
+foo: 1
+bar: true
+xyz: ['1', '2']
+empty_arr: []
+abc:
+  def: test
+  qwe: 1
+  zyx: { val: 1 }
+pom.bom: 3

packages/kbn-apm-config-loader/__fixtures__/two.yml

@@ -0,0 +1,10 @@
+foo: 2
+baz: bonkers
+xyz: ['3', '4']
+arr: [1]
+empty_arr: []
+abc:
+  ghi: test2
+  qwe: 2
+  zyx: {}
+pom.mob: 4

packages/kbn-apm-config-loader/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@kbn/apm-config-loader",
+  "main": "./target/index.js",
+  "types": "./target/index.d.ts",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "dependencies": {
+    "@elastic/safer-lodash-set": "0.0.0",
+    "@kbn/utils": "1.0.0",
+    "js-yaml": "3.13.1",
+    "lodash": "^4.17.20"
+  },
+  "devDependencies": {
+    "typescript": "4.0.2",
+    "tsd": "^0.7.4"
+  }
+}

packages/kbn-apm-config-loader/src/config.test.mocks.ts

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { join } from 'path';
+const childProcessModule = jest.requireActual('child_process');
+const fsModule = jest.requireActual('fs');
+
+export const mockedRootDir = '/root';
+
+export const packageMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true });
+
+export const devConfigMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, {
+  virtual: true,
+});
+
+export const gitRevExecMock = jest.fn();
+jest.doMock('child_process', () => ({
+  ...childProcessModule,
+  execSync: (command: string, options: any) => {
+    if (command.startsWith('git rev-parse')) {
+      return gitRevExecMock(command, options);
+    }
+    return childProcessModule.execSync(command, options);
+  },
+}));
+
+export const readUuidFileMock = jest.fn();
+jest.doMock('fs', () => ({
+  ...fsModule,
+  readFileSync: (path: string, options: any) => {
+    if (path.endsWith('uuid')) {
+      return readUuidFileMock(path, options);
+    }
+    return fsModule.readFileSync(path, options);
+  },
+}));
+
+export const resetAllMocks = () => {
+  packageMock.raw = {};
+  devConfigMock.raw = {};
+  gitRevExecMock.mockReset();
+  readUuidFileMock.mockReset();
+  jest.resetModules();
+};

packages/kbn-apm-config-loader/src/config.test.ts

@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  packageMock,
+  mockedRootDir,
+  gitRevExecMock,
+  devConfigMock,
+  readUuidFileMock,
+  resetAllMocks,
+} from './config.test.mocks';
+
+import { ApmConfiguration } from './config';
+
+describe('ApmConfiguration', () => {
+  beforeEach(() => {
+    packageMock.raw = {
+      version: '8.0.0',
+      build: {
+        sha: 'sha',
+      },
+    };
+  });
+
+  afterEach(() => {
+    resetAllMocks();
+  });
+
+  it('sets the correct service name', () => {
+    packageMock.raw = {
+      version: '9.2.1',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('myservice').serviceName).toBe('myservice-9_2_1');
+  });
+
+  it('sets the git revision from `git rev-parse` command in non distribution mode', () => {
+    gitRevExecMock.mockReturnValue('some-git-rev');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('some-git-rev');
+  });
+
+  it('sets the git revision from `pkg.build.sha` in distribution mode', () => {
+    gitRevExecMock.mockReturnValue('dev-sha');
+    packageMock.raw = {
+      version: '9.2.1',
+      build: {
+        sha: 'distribution-sha',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('distribution-sha');
+  });
+
+  it('reads the kibana uuid from the uuid file', () => {
+    readUuidFileMock.mockReturnValue('instance-uuid');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('instance-uuid');
+  });
+
+  it('uses the uuid from the kibana config if present', () => {
+    readUuidFileMock.mockReturnValue('uuid-from-file');
+    const kibanaConfig = {
+      server: {
+        uuid: 'uuid-from-config',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('uuid-from-config');
+  });
+
+  it('uses the correct default config depending on the `isDistributable` parameter', () => {
+    let config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        serverUrl: expect.any(String),
+        secretToken: expect.any(String),
+      })
+    );
+
+    config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl');
+  });
+
+  it('loads the configuration from the kibana config file', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://url',
+        secretToken: 'secret',
+      })
+    );
+  });
+
+  it('loads the configuration from the dev config is present', () => {
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+      })
+    );
+  });
+
+  it('respect the precedence of the dev config', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+        secretToken: 'secret',
+      })
+    );
+  });
+});