Skip to content

Commit f0df061

Browse files
mhliddmhlidd
authored
Adding Span Link support for distributed tracing header extractions with invalid traces (#4874)
* initial commit * updating _links and when links are created * logging * add link to instrumentation * updating integrations to include span links * fixing syntax error * fixing ci tests * updating unit test * fix ci * fixing moleculer tests * safe checking all contexts before getting links
1 parent 61c5a32 commit f0df061

File tree

17 files changed

+90
-42
lines changed

17 files changed

+90
-42
lines changed

‎packages/datadog-plugin-amqplib/src/consumer.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ class AmqplibConsumerPlugin extends ConsumerPlugin {
2626
'amqp.consumerTag': fields.consumerTag,
2727
'amqp.source': fields.source,
2828
'amqp.destination': fields.destination
29-
}
29+
},
30+
extractedLinks: childOf?._links
3031
})
3132

3233
if (

‎packages/datadog-plugin-aws-sdk/src/services/kinesis.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ class Kinesis extends BaseAwsSdkPlugin {
4242
{},
4343
this.requestTags.get(request) || {},
4444
{ 'span.kind': 'server' }
45-
)
45+
),
46+
extractedLinks: responseExtraction.maybeChildOf._links
4647
}
4748
span = plugin.tracer.startSpan('aws.response', options)
4849
this.enter(span, store)

‎packages/datadog-plugin-aws-sdk/src/services/sqs.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ class Sqs extends BaseAwsSdkPlugin {
3333
{},
3434
this.requestTags.get(request) || {},
3535
{ 'span.kind': 'server' }
36-
)
36+
),
37+
extractedLinks: contextExtraction.datadogContext._links
3738
}
3839
parsedMessageAttributes = contextExtraction.parsedAttributes
3940
span = plugin.tracer.startSpan('aws.response', options)

‎packages/datadog-plugin-child_process/test/index.spec.js

+12-6
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ describe('Child process plugin', () => {
6262
'span.type': 'system',
6363
'cmd.exec': JSON.stringify(['ls', '-l'])
6464
},
65-
integrationName: 'system'
65+
integrationName: 'system',
66+
links: undefined
6667
}
6768
)
6869
})
@@ -84,7 +85,8 @@ describe('Child process plugin', () => {
8485
'span.type': 'system',
8586
'cmd.shell': 'ls -l'
8687
},
87-
integrationName: 'system'
88+
integrationName: 'system',
89+
links: undefined
8890
}
8991
)
9092
})
@@ -109,7 +111,8 @@ describe('Child process plugin', () => {
109111
'cmd.exec': JSON.stringify(['echo', arg, '']),
110112
'cmd.truncated': 'true'
111113
},
112-
integrationName: 'system'
114+
integrationName: 'system',
115+
links: undefined
113116
}
114117
)
115118
})
@@ -134,7 +137,8 @@ describe('Child process plugin', () => {
134137
'cmd.shell': 'ls -l /h ',
135138
'cmd.truncated': 'true'
136139
},
137-
integrationName: 'system'
140+
integrationName: 'system',
141+
links: undefined
138142
}
139143
)
140144
})
@@ -160,7 +164,8 @@ describe('Child process plugin', () => {
160164
'cmd.exec': JSON.stringify(['ls', '-l', '', '']),
161165
'cmd.truncated': 'true'
162166
},
163-
integrationName: 'system'
167+
integrationName: 'system',
168+
links: undefined
164169
}
165170
)
166171
})
@@ -186,7 +191,8 @@ describe('Child process plugin', () => {
186191
'cmd.shell': 'ls -l /home -t',
187192
'cmd.truncated': 'true'
188193
},
189-
integrationName: 'system'
194+
integrationName: 'system',
195+
links: undefined
190196
}
191197
)
192198
})

‎packages/datadog-plugin-google-cloud-pubsub/src/consumer.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ class GoogleCloudPubsubConsumerPlugin extends ConsumerPlugin {
2222
},
2323
metrics: {
2424
'pubsub.ack': 0
25-
}
25+
},
26+
extractedLinks: childOf?._links
2627
})
2728
if (this.config.dsmEnabled && message?.attributes) {
2829
const payloadSize = getMessageSize(message)

‎packages/datadog-plugin-grpc/src/server.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ class GrpcServerPlugin extends ServerPlugin {
4848
},
4949
metrics: {
5050
'grpc.status.code': 0
51-
}
51+
},
52+
extractedLinks: childOf?._links
5253
})
5354

5455
addMetadataTags(span, metadata, metadataFilter, 'request')

‎packages/datadog-plugin-jest/src/index.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ class JestPlugin extends CiPlugin {
219219
[COMPONENT]: this.constructor.id,
220220
...this.testEnvironmentMetadata,
221221
...testSuiteMetadata
222-
}
222+
},
223+
extractedLinks: testSessionSpanContext?._links
223224
})
224225
this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
225226
if (_ddTestCodeCoverageEnabled) {

‎packages/datadog-plugin-kafkajs/src/consumer.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ class KafkajsConsumerPlugin extends ConsumerPlugin {
7676
},
7777
metrics: {
7878
'kafka.partition': partition
79-
}
79+
},
80+
extractedLinks: childOf?._links
8081
})
8182
if (this.config.dsmEnabled && message?.headers) {
8283
const payloadSize = getMessageSize(message)

‎packages/datadog-plugin-moleculer/src/server.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ class MoleculerServerPlugin extends ServerPlugin {
99

1010
start ({ action, ctx, broker }) {
1111
const followsFrom = this.tracer.extract('text_map', ctx.meta)
12-
1312
this.startSpan(this.operationName(), {
1413
childOf: followsFrom || this.activeSpan,
1514
service: this.config.service || this.serviceName(),
@@ -19,7 +18,8 @@ class MoleculerServerPlugin extends ServerPlugin {
1918
meta: {
2019
'resource.name': action.name,
2120
...moleculerTags(broker, ctx, this.config)
22-
}
21+
},
22+
extractedLinks: followsFrom?._links
2323
})
2424
}
2525
}

‎packages/datadog-plugin-rhea/src/consumer.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ class RheaConsumerPlugin extends ConsumerPlugin {
2828
component: 'rhea',
2929
'amqp.link.source.address': name,
3030
'amqp.link.role': 'receiver'
31-
}
31+
},
32+
extractedLinks: childOf?._links
3233
})
3334

3435
if (

‎packages/datadog-plugin-vitest/src/index.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ class VitestPlugin extends CiPlugin {
191191
[COMPONENT]: this.constructor.id,
192192
...this.testEnvironmentMetadata,
193193
...testSuiteMetadata
194-
}
194+
},
195+
extractedLinks: testSessionSpanContext?._links
195196
})
196197
this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
197198
const store = storage.getStore()

‎packages/dd-trace/src/opentracing/propagation/text_map.js

+34-21
Original file line numberDiff line numberDiff line change
@@ -290,50 +290,63 @@ class TextMapPropagator {
290290
}
291291

292292
_extractSpanContext (carrier) {
293-
let spanContext = null
293+
let context = null
294294
for (const extractor of this._config.tracePropagationStyle.extract) {
295-
// add logic to ensure tracecontext headers takes precedence over other extracted headers
296-
if (spanContext !== null) {
297-
if (this._config.tracePropagationExtractFirst) {
298-
return spanContext
299-
}
300-
if (extractor !== 'tracecontext') {
301-
continue
302-
}
303-
spanContext = this._resolveTraceContextConflicts(
304-
this._extractTraceparentContext(carrier), spanContext, carrier)
305-
break
306-
}
307-
295+
let extractedContext = null
308296
switch (extractor) {
309297
case 'datadog':
310-
spanContext = this._extractDatadogContext(carrier)
298+
extractedContext = this._extractDatadogContext(carrier)
311299
break
312300
case 'tracecontext':
313-
spanContext = this._extractTraceparentContext(carrier)
301+
extractedContext = this._extractTraceparentContext(carrier)
314302
break
315303
case 'b3' && this
316304
._config
317305
.tracePropagationStyle
318306
.otelPropagators: // TODO: should match "b3 single header" in next major
319307
case 'b3 single header': // TODO: delete in major after singular "b3"
320-
spanContext = this._extractB3SingleContext(carrier)
308+
extractedContext = this._extractB3SingleContext(carrier)
321309
break
322310
case 'b3':
323311
case 'b3multi':
324-
spanContext = this._extractB3MultiContext(carrier)
312+
extractedContext = this._extractB3MultiContext(carrier)
325313
break
326314
default:
327315
if (extractor !== 'baggage') log.warn(`Unknown propagation style: ${extractor}`)
328316
}
329317

318+
if (extractedContext === null) { // If the current extractor was invalid, continue to the next extractor
319+
continue
320+
}
321+
322+
if (context === null) {
323+
context = extractedContext
324+
if (this._config.tracePropagationExtractFirst) {
325+
return context
326+
}
327+
} else {
328+
// If extractor is tracecontext, add tracecontext specific information to the context
329+
if (extractor === 'tracecontext') {
330+
context = this._resolveTraceContextConflicts(
331+
this._extractTraceparentContext(carrier), context, carrier)
332+
}
333+
if (extractedContext._traceId && extractedContext._spanId &&
334+
extractedContext.toTraceId(true) !== context.toTraceId(true)) {
335+
const link = {
336+
context: extractedContext,
337+
attributes: { reason: 'terminated_context', context_headers: extractor }
338+
}
339+
context._links.push(link)
340+
}
341+
}
342+
330343
if (this._config.tracePropagationStyle.extract.includes('baggage') && carrier.baggage) {
331-
spanContext = spanContext || new DatadogSpanContext()
332-
this._extractBaggageItems(carrier, spanContext)
344+
context = context || new DatadogSpanContext()
345+
this._extractBaggageItems(carrier, context)
333346
}
334347
}
335348

336-
return spanContext || this._extractSqsdContext(carrier)
349+
return context || this._extractSqsdContext(carrier)
337350
}
338351

339352
_extractDatadogContext (carrier) {

‎packages/dd-trace/src/opentracing/span_context.js

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ class DatadogSpanContext {
1818
this._tags = props.tags || {}
1919
this._sampling = props.sampling || {}
2020
this._spanSampling = undefined
21+
this._links = props.links || []
2122
this._baggageItems = props.baggageItems || {}
2223
this._traceparent = props.traceparent
2324
this._tracestate = props.tracestate

‎packages/dd-trace/src/plugins/tracing.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,8 @@ class TracingPlugin extends Plugin {
101101
}
102102
}
103103

104-
startSpan (name, { childOf, kind, meta, metrics, service, resource, type } = {}, enter = true) {
104+
startSpan (name, { childOf, kind, meta, metrics, service, resource, type, extractedLinks } = {}, enter = true) {
105105
const store = storage.getStore()
106-
107106
if (store && childOf === undefined) {
108107
childOf = store.span
109108
}
@@ -119,7 +118,8 @@ class TracingPlugin extends Plugin {
119118
...meta,
120119
...metrics
121120
},
122-
integrationName: type
121+
integrationName: type,
122+
links: extractedLinks
123123
})
124124

125125
analyticsSampler.sample(span, this.config.measured)

‎packages/dd-trace/src/plugins/util/web.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ const web = {
267267
}
268268
}
269269

270-
const span = tracer.startSpan(name, { childOf })
270+
const span = tracer.startSpan(name, { childOf, extractedLinks: childOf?.links })
271271

272272
return span
273273
},

‎packages/dd-trace/test/opentracing/propagation/text_map.spec.js

+17
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,23 @@ describe('TextMapPropagator', () => {
692692
}
693693
})
694694

695+
it('should create span links when traces have inconsistent traceids', () => {
696+
// Add a traceparent header and it will prioritize it
697+
const traceId = '1111aaaa2222bbbb3333cccc4444dddd'
698+
const spanId = '5555eeee6666ffff'
699+
textMap.traceparent = `00-${traceId}-${spanId}-01`
700+
701+
config.tracePropagationStyle.extract = ['tracecontext', 'datadog']
702+
703+
const first = propagator.extract(textMap)
704+
705+
expect(first._links.length).to.equal(1)
706+
expect(first._links[0].context.toTraceId()).to.equal(textMap['x-datadog-trace-id'])
707+
expect(first._links[0].context.toSpanId()).to.equal(textMap['x-datadog-parent-id'])
708+
expect(first._links[0].attributes.reason).to.equal('terminated_context')
709+
expect(first._links[0].attributes.context_headers).to.equal('datadog')
710+
})
711+
695712
describe('with B3 propagation as multiple headers', () => {
696713
beforeEach(() => {
697714
config.tracePropagationStyle.extract = ['b3multi']

‎packages/dd-trace/test/opentracing/span_context.spec.js

+2
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ describe('SpanContext', () => {
4848
_tags: {},
4949
_sampling: { priority: 2 },
5050
_spanSampling: undefined,
51+
_links: [],
5152
_baggageItems: { foo: 'bar' },
5253
_noop: noop,
5354
_trace: {
@@ -77,6 +78,7 @@ describe('SpanContext', () => {
7778
_tags: {},
7879
_sampling: {},
7980
_spanSampling: undefined,
81+
_links: [],
8082
_baggageItems: {},
8183
_noop: null,
8284
_trace: {

0 commit comments

Comments
 (0)