Skip to content

Commit d1a5262

Browse files
ps48aaarone90
ps48
authored and
aaarone90
committed
Support custom logs correlation (#2375)
* support custom logs correlation Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> * add support for custom field mappings in logs Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> * update explorer fields Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> * add support for custom timestamp field Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> --------- Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> Signed-off-by: Aaron Alvarez <aaarone@amazon.com>
1 parent 0e0bb31 commit d1a5262

File tree

10 files changed

+340
-41
lines changed

10 files changed

+340
-41
lines changed

‎common/constants/trace_analytics.ts

+12
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,18 @@ export const TRACE_ANALYTICS_DSL_ROUTE = '/api/observability/trace_analytics/que
2525
export const TRACE_CUSTOM_SPAN_INDEX_SETTING = 'observability:traceAnalyticsSpanIndices';
2626
export const TRACE_CUSTOM_SERVICE_INDEX_SETTING = 'observability:traceAnalyticsServiceIndices';
2727
export const TRACE_CUSTOM_MODE_DEFAULT_SETTING = 'observability:traceAnalyticsCustomModeDefault';
28+
export const TRACE_CORRELATED_LOGS_INDEX_SETTING =
29+
'observability:traceAnalyticsCorrelatedLogsIndices';
30+
export const TRACE_LOGS_FIELD_MAPPNIGS_SETTING =
31+
'observability:traceAnalyticsCorrelatedLogsFieldMappings';
32+
33+
export const DEFAULT_SS4O_LOGS_INDEX = 'ss4o_logs-*';
34+
export const DEFAULT_CORRELATED_LOGS_FIELD_MAPPINGS = `
35+
{
36+
"serviceName": "serviceName",
37+
"spanId": "spanId",
38+
"timestamp": "time"
39+
}`;
2840

2941
export enum TRACE_TABLE_TITLES {
3042
all_spans = 'All Spans',

‎common/types/trace_analytics.ts

+6
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,9 @@ export interface GraphVisEdge {
5757

5858
export type TraceAnalyticsMode = 'jaeger' | 'data_prepper' | 'custom_data_prepper';
5959
export type TraceQueryMode = keyof typeof TRACE_TABLE_TITLES;
60+
61+
export interface CorrelatedLogsFieldMappings {
62+
serviceName: string;
63+
spanId: string;
64+
timestamp: string;
65+
}

‎public/components/event_analytics/explorer/explorer.tsx

+11-2
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ import {
111111
import { getVizContainerProps } from '../../visualizations/charts/helpers';
112112
import { TabContext, useFetchEvents, useFetchPatterns, useFetchVisualizations } from '../hooks';
113113
import {
114-
render as updateCountDistribution,
115114
selectCountDistribution,
115+
render as updateCountDistribution,
116116
} from '../redux/slices/count_distribution_slice';
117117
import { selectFields, updateFields } from '../redux/slices/field_slice';
118118
import { selectQueryResult } from '../redux/slices/query_result_slice';
@@ -122,8 +122,8 @@ import { selectExplorerVisualization } from '../redux/slices/visualization_slice
122122
import {
123123
change as changeVisualizationConfig,
124124
change as changeVizConfig,
125-
change as updateVizConfig,
126125
selectVisualizationConfig,
126+
change as updateVizConfig,
127127
} from '../redux/slices/viualization_config_slice';
128128
import { getDefaultVisConfig } from '../utils';
129129
import { formatError, getContentTabTitle } from '../utils/utils';
@@ -274,6 +274,7 @@ export const Explorer = ({
274274
datasourceName,
275275
datasourceType,
276276
queryToRun,
277+
timestampField,
277278
startTimeRange,
278279
endTimeRange,
279280
}: any = historyFromRedirection.location.state;
@@ -295,6 +296,14 @@ export const Explorer = ({
295296
})
296297
);
297298
}
299+
if (timestampField) {
300+
dispatch(
301+
changeQuery({
302+
tabId,
303+
query: { [SELECTED_TIMESTAMP]: timestampField },
304+
})
305+
);
306+
}
298307
if (queryToRun) {
299308
dispatch(
300309
changeQuery({

‎public/components/trace_analytics/components/common/__tests__/custom_index_flyout.test.tsx

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
7+
import React from 'react';
8+
import { useToast } from '../../../../common/toast';
9+
import { CustomIndexFlyout } from '../custom_index_flyout';
10+
import { TraceSettings } from '../helper_functions';
11+
12+
// Mock TraceSettings functions
13+
jest.mock('../helper_functions', () => ({
14+
TraceSettings: {
15+
getCustomSpanIndex: jest.fn(),
16+
getCustomServiceIndex: jest.fn(),
17+
getCorrelatedLogsIndex: jest.fn(),
18+
getCorrelatedLogsFieldMappings: jest.fn(),
19+
getCustomModeSetting: jest.fn(),
20+
setCustomSpanIndex: jest.fn(),
21+
setCustomServiceIndex: jest.fn(),
22+
setCorrelatedLogsIndex: jest.fn(),
23+
setCorrelatedLogsFieldMappings: jest.fn(),
24+
setCustomModeSetting: jest.fn(),
25+
},
26+
}));
27+
28+
// Mock the toast hook
29+
jest.mock('../../../../common/toast', () => ({
30+
useToast: jest.fn(),
31+
}));
32+
33+
describe('CustomIndexFlyout test', () => {
34+
let setIsFlyoutVisibleMock: jest.Mock;
35+
let setToastMock: jest.Mock;
36+
37+
beforeEach(() => {
38+
setIsFlyoutVisibleMock = jest.fn();
39+
setToastMock = jest.fn();
40+
41+
(useToast as jest.Mock).mockReturnValue({ setToast: setToastMock });
42+
43+
(TraceSettings.getCustomSpanIndex as jest.Mock).mockReturnValue('span-index-1');
44+
(TraceSettings.getCustomServiceIndex as jest.Mock).mockReturnValue('service-index-1');
45+
(TraceSettings.getCorrelatedLogsIndex as jest.Mock).mockReturnValue('logs-index-1');
46+
(TraceSettings.getCorrelatedLogsFieldMappings as jest.Mock).mockReturnValue({
47+
serviceName: 'service_field',
48+
spanId: 'span_field',
49+
timestamp: 'timestamp_field',
50+
});
51+
(TraceSettings.getCustomModeSetting as jest.Mock).mockReturnValue(false);
52+
});
53+
54+
it('renders flyout when isFlyoutVisible is true', () => {
55+
render(
56+
<CustomIndexFlyout isFlyoutVisible={true} setIsFlyoutVisible={setIsFlyoutVisibleMock} />
57+
);
58+
59+
expect(screen.getByText('Manage custom source')).toBeInTheDocument();
60+
expect(screen.getByLabelText('spanIndices')).toHaveValue('span-index-1');
61+
expect(screen.getByLabelText('serviceIndices')).toHaveValue('service-index-1');
62+
expect(screen.getByLabelText('logsIndices')).toHaveValue('logs-index-1');
63+
expect(screen.getByLabelText('Enable custom source as default mode')).not.toBeChecked();
64+
});
65+
66+
it('updates span indices when input changes', () => {
67+
render(
68+
<CustomIndexFlyout isFlyoutVisible={true} setIsFlyoutVisible={setIsFlyoutVisibleMock} />
69+
);
70+
71+
const input = screen.getByLabelText('Custom span indices');
72+
fireEvent.change(input, { target: { value: 'new-span-index' } });
73+
74+
expect(input).toHaveValue('new-span-index');
75+
});
76+
77+
it('calls TraceSettings set functions when save button is clicked', async () => {
78+
render(
79+
<CustomIndexFlyout isFlyoutVisible={true} setIsFlyoutVisible={setIsFlyoutVisibleMock} />
80+
);
81+
82+
const saveButton = screen.getByText('Save');
83+
fireEvent.click(saveButton);
84+
85+
await waitFor(() => {
86+
expect(TraceSettings.setCustomSpanIndex).toHaveBeenCalledWith('span-index-1');
87+
expect(TraceSettings.setCustomServiceIndex).toHaveBeenCalledWith('service-index-1');
88+
expect(TraceSettings.setCorrelatedLogsIndex).toHaveBeenCalledWith('logs-index-1');
89+
expect(TraceSettings.setCorrelatedLogsFieldMappings).toHaveBeenCalled();
90+
expect(TraceSettings.setCustomModeSetting).toHaveBeenCalledWith(false);
91+
expect(setToastMock).toHaveBeenCalledWith(
92+
'Updated trace analytics settings successfully',
93+
'success'
94+
);
95+
});
96+
});
97+
98+
it('shows error toast if save settings fail', async () => {
99+
(TraceSettings.setCustomSpanIndex as jest.Mock).mockRejectedValue(new Error('Save error'));
100+
101+
render(
102+
<CustomIndexFlyout isFlyoutVisible={true} setIsFlyoutVisible={setIsFlyoutVisibleMock} />
103+
);
104+
105+
const saveButton = screen.getByText('Save');
106+
fireEvent.click(saveButton);
107+
108+
await waitFor(() => {
109+
expect(setToastMock).toHaveBeenCalledWith(
110+
'Failed to update trace analytics settings',
111+
'danger'
112+
);
113+
});
114+
});
115+
116+
it('closes the flyout when Close button is clicked', () => {
117+
render(
118+
<CustomIndexFlyout isFlyoutVisible={true} setIsFlyoutVisible={setIsFlyoutVisibleMock} />
119+
);
120+
121+
const closeButton = screen.getByText('Close');
122+
fireEvent.click(closeButton);
123+
124+
expect(setIsFlyoutVisibleMock).toHaveBeenCalledWith(false);
125+
});
126+
});

‎public/components/trace_analytics/components/common/custom_index_flyout.tsx

+81-11
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@ import {
2121
EuiTitle,
2222
} from '@elastic/eui';
2323
import React, { Fragment, useEffect, useState } from 'react';
24-
import {
25-
TRACE_CUSTOM_SERVICE_INDEX_SETTING,
26-
TRACE_CUSTOM_SPAN_INDEX_SETTING,
27-
TRACE_CUSTOM_MODE_DEFAULT_SETTING,
28-
} from '../../../../../common/constants/trace_analytics';
24+
import { DEFAULT_CORRELATED_LOGS_FIELD_MAPPINGS } from '../../../../../common/constants/trace_analytics';
25+
import { CorrelatedLogsFieldMappings } from '../../../../../common/types/trace_analytics';
2926
import { uiSettingsService } from '../../../../../common/utils';
3027
import { useToast } from '../../../common/toast';
28+
import { TraceSettings } from './helper_functions';
3129

3230
interface CustomIndexFlyoutProps {
3331
isFlyoutVisible: boolean;
@@ -42,6 +40,10 @@ export const CustomIndexFlyout = ({
4240
const [spanIndices, setSpanIndices] = useState('');
4341
const [serviceIndices, setServiceIndices] = useState('');
4442
const [customModeDefault, setCustomModeDefault] = useState(false);
43+
const [correlatedLogsIndices, setCorrelatedLogsIndices] = useState('');
44+
const [correlatedLogsFieldMappings, setCorrelatedLogsFieldMappings] = useState(
45+
{} as CorrelatedLogsFieldMappings
46+
);
4547
const [isLoading, setIsLoading] = useState(false);
4648

4749
const onChangeSpanIndices = (e: { target: { value: React.SetStateAction<string> } }) => {
@@ -52,22 +54,41 @@ export const CustomIndexFlyout = ({
5254
setServiceIndices(e.target.value);
5355
};
5456

57+
const onChangeCorrelatedLogsIndices = (e: {
58+
target: { value: React.SetStateAction<string> };
59+
}) => {
60+
setCorrelatedLogsIndices(e.target.value);
61+
};
62+
63+
const onChangeLogsFieldMappings = (e: React.ChangeEvent<HTMLInputElement>) => {
64+
const { name, value } = e.target;
65+
66+
setCorrelatedLogsFieldMappings((prevMappings) => ({
67+
...prevMappings,
68+
[name]: value,
69+
}));
70+
};
71+
5572
const onToggleCustomModeDefault = (e: { target: { checked: boolean } }) => {
5673
setCustomModeDefault(e.target.checked);
5774
};
5875

5976
useEffect(() => {
60-
setSpanIndices(uiSettingsService.get(TRACE_CUSTOM_SPAN_INDEX_SETTING));
61-
setServiceIndices(uiSettingsService.get(TRACE_CUSTOM_SERVICE_INDEX_SETTING));
62-
setCustomModeDefault(uiSettingsService.get(TRACE_CUSTOM_MODE_DEFAULT_SETTING) || false);
77+
setSpanIndices(TraceSettings.getCustomSpanIndex());
78+
setServiceIndices(TraceSettings.getCustomServiceIndex());
79+
setCorrelatedLogsIndices(TraceSettings.getCorrelatedLogsIndex());
80+
setCorrelatedLogsFieldMappings(TraceSettings.getCorrelatedLogsFieldMappings());
81+
setCustomModeDefault(TraceSettings.getCustomModeSetting());
6382
}, [uiSettingsService]);
6483

6584
const onSaveSettings = async () => {
6685
try {
6786
setIsLoading(true);
68-
await uiSettingsService.set(TRACE_CUSTOM_SPAN_INDEX_SETTING, spanIndices);
69-
await uiSettingsService.set(TRACE_CUSTOM_SERVICE_INDEX_SETTING, serviceIndices);
70-
await uiSettingsService.set(TRACE_CUSTOM_MODE_DEFAULT_SETTING, customModeDefault);
87+
await TraceSettings.setCustomSpanIndex(spanIndices);
88+
await TraceSettings.setCustomServiceIndex(serviceIndices);
89+
await TraceSettings.setCorrelatedLogsIndex(correlatedLogsIndices);
90+
await TraceSettings.setCorrelatedLogsFieldMappings(correlatedLogsFieldMappings);
91+
await TraceSettings.setCustomModeSetting(customModeDefault);
7192
setIsLoading(false);
7293
setToast('Updated trace analytics settings successfully', 'success');
7394
} catch (error) {
@@ -77,6 +98,25 @@ export const CustomIndexFlyout = ({
7798
setIsLoading(false);
7899
};
79100

101+
const correlatedFieldsForm = () => {
102+
const correlatedFieldsFromSettings: CorrelatedLogsFieldMappings = TraceSettings.getCorrelatedLogsFieldMappings();
103+
return Object.keys(correlatedFieldsFromSettings).map((key) => {
104+
return (
105+
<>
106+
<EuiFormRow label={key}>
107+
<EuiCompressedFieldText
108+
name={key}
109+
aria-label={key}
110+
placeholder={JSON.parse(DEFAULT_CORRELATED_LOGS_FIELD_MAPPINGS)[key]}
111+
value={correlatedLogsFieldMappings[key]}
112+
onChange={onChangeLogsFieldMappings}
113+
/>
114+
</EuiFormRow>
115+
</>
116+
);
117+
});
118+
};
119+
80120
const callout = (
81121
<EuiCallOut
82122
title="Custom source in trace analytics is an experimental feature"
@@ -143,6 +183,36 @@ export const CustomIndexFlyout = ({
143183
/>
144184
</EuiFormRow>
145185
</EuiDescribedFormGroup>
186+
<EuiDescribedFormGroup
187+
title={<h3>Correlated logs indices</h3>}
188+
description={
189+
<Fragment>
190+
Configure custom logs indices to be used by the trace analytics plugin to correlate
191+
spans and services
192+
</Fragment>
193+
}
194+
>
195+
<EuiFormRow label="Correlated logs indices">
196+
<EuiCompressedFieldText
197+
name="logsIndices"
198+
aria-label="logsIndices"
199+
placeholder="index1"
200+
value={correlatedLogsIndices}
201+
onChange={onChangeCorrelatedLogsIndices}
202+
/>
203+
</EuiFormRow>
204+
</EuiDescribedFormGroup>
205+
<EuiDescribedFormGroup
206+
title={<h3>Correlated logs fields</h3>}
207+
description={
208+
<Fragment>
209+
Configure correlated logs fields, to be used by the trace analytics plugin for
210+
correlate spans and services to logs
211+
</Fragment>
212+
}
213+
>
214+
{correlatedFieldsForm()}
215+
</EuiDescribedFormGroup>
146216
<EuiDescribedFormGroup
147217
title={<h3>Set default mode</h3>}
148218
description={

0 commit comments

Comments
 (0)