add span events as a top level field for v0.4 encoding (#5229)

* adds span events at top level for v0.4 encoding

* only serializes span events as top level if DD_TRACE_NATIVE_SPAN_EVENTS=true

Co-authored-by: Thomas Hunter II <tlhunter@datadog.com>

Author: 2 people

packages/dd-trace/src/config.js

@@ -595,6 +595,7 @@ class Config {
     this._setValue(defaults, 'vertexai.spanCharLimit', 128)
     this._setValue(defaults, 'vertexai.spanPromptCompletionSampleRate', 1.0)
     this._setValue(defaults, 'trace.aws.addSpanPointers', true)
+    this._setValue(defaults, 'trace.nativeSpanEvents', false)
   }
 
   _applyLocalStableConfig () {
@@ -765,6 +766,7 @@ class Config {
       DD_VERTEXAI_SPAN_PROMPT_COMPLETION_SAMPLE_RATE,
       DD_VERTEXAI_SPAN_CHAR_LIMIT,
       DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED,
+      DD_TRACE_NATIVE_SPAN_EVENTS,
       OTEL_METRICS_EXPORTER,
       OTEL_PROPAGATORS,
       OTEL_RESOURCE_ATTRIBUTES,
@@ -977,6 +979,7 @@ class Config {
     this._setBoolean(env, 'trace.aws.addSpanPointers', DD_TRACE_AWS_ADD_SPAN_POINTERS)
     this._setString(env, 'trace.dynamoDb.tablePrimaryKeys', DD_TRACE_DYNAMODB_TABLE_PRIMARY_KEYS)
     this._setArray(env, 'graphqlErrorExtensions', DD_TRACE_GRAPHQL_ERROR_EXTENSIONS)
+    this._setBoolean(env, 'trace.nativeSpanEvents', DD_TRACE_NATIVE_SPAN_EVENTS)
     this._setValue(
       env,
       'vertexai.spanPromptCompletionSampleRate',
@@ -1114,6 +1117,7 @@ class Config {
     this._setString(opts, 'version', options.version || tags.version)
     this._setBoolean(opts, 'inferredProxyServicesEnabled', options.inferredProxyServicesEnabled)
     this._setBoolean(opts, 'graphqlErrorExtensions', options.graphqlErrorExtensions)
+    this._setBoolean(opts, 'trace.nativeSpanEvents', options.trace?.nativeSpanEvents)
 
     // For LLMObs, we want the environment variable to take precedence over the options.
     // This is reliant on environment config being set before options.

packages/dd-trace/src/encode/0.4.js

@@ -5,11 +5,22 @@ const { Chunk, MsgpackEncoder } = require('../msgpack')
 const log = require('../log')
 const { isTrue } = require('../util')
 const coalesce = require('koalas')
+const { memoize } = require('../log/utils')
 
 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB
 
-function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span, false))
+function formatSpan (span, config) {
+  span = normalizeSpan(truncateSpan(span, false))
+  if (span.span_events) {
+    // ensure span events are encoded as tags if agent doesn't support native top level span events
+    if (!config?.trace?.nativeSpanEvents) {
+      span.meta.events = JSON.stringify(span.span_events)
+      delete span.span_events
+    } else {
+      formatSpanEvents(span)
+    }
+  }
+  return span
 }
 
 class AgentEncoder {
@@ -24,6 +35,7 @@ class AgentEncoder {
       process.env.DD_TRACE_ENCODING_DEBUG,
       false
     ))
+    this._config = this._writer?._config
   }
 
   count () {
@@ -74,16 +86,18 @@ class AgentEncoder {
     this._encodeArrayPrefix(bytes, trace)
 
     for (let span of trace) {
-      span = formatSpan(span)
+      span = formatSpan(span, this._config)
       bytes.reserve(1)
 
-      if (span.type && span.meta_struct) {
-        bytes.buffer[bytes.length - 1] = 0x8d
-      } else if (span.type || span.meta_struct) {
-        bytes.buffer[bytes.length - 1] = 0x8c
-      } else {
-        bytes.buffer[bytes.length - 1] = 0x8b
-      }
+      // this is the original size of the fixed map for span attributes that always exist
+      let mapSize = 11
+
+      // increment the payload map size depending on if some optional fields exist
+      if (span.type) mapSize += 1
+      if (span.meta_struct) mapSize += 1
+      if (span.span_events) mapSize += 1
+
+      bytes.buffer[bytes.length - 1] = 0x80 + mapSize
 
       if (span.type) {
         this._encodeString(bytes, 'type')
@@ -112,6 +126,10 @@ class AgentEncoder {
       this._encodeMap(bytes, span.meta)
       this._encodeString(bytes, 'metrics')
       this._encodeMap(bytes, span.metrics)
+      if (span.span_events) {
+        this._encodeString(bytes, 'span_events')
+        this._encodeObjectAsArray(bytes, span.span_events, new Set())
+      }
       if (span.meta_struct) {
         this._encodeString(bytes, 'meta_struct')
         this._encodeMetaStruct(bytes, span.meta_struct)
@@ -200,6 +218,9 @@ class AgentEncoder {
       case 'number':
         this._encodeFloat(bytes, value)
         break
+      case 'boolean':
+        this._encodeBool(bytes, value)
+        break
       default:
         // should not happen
     }
@@ -258,7 +279,7 @@ class AgentEncoder {
       this._encodeObjectAsArray(bytes, value, circularReferencesDetector)
     } else if (value !== null && typeof value === 'object') {
       this._encodeObjectAsMap(bytes, value, circularReferencesDetector)
-    } else if (typeof value === 'string' || typeof value === 'number') {
+    } else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
       this._encodeValue(bytes, value)
     }
   }
@@ -268,7 +289,7 @@ class AgentEncoder {
     const validKeys = keys.filter(key => {
       const v = value[key]
       return typeof v === 'string' ||
-        typeof v === 'number' ||
+        typeof v === 'number' || typeof v === 'boolean' ||
         (v !== null && typeof v === 'object' && !circularReferencesDetector.has(v))
     })
 
@@ -319,4 +340,79 @@ class AgentEncoder {
   }
 }
 
+const memoizedLogDebug = memoize((key, message) => {
+  log.debug(message)
+  // return something to store in memoize cache
+  return true
+})
+
+function formatSpanEvents (span) {
+  for (const spanEvent of span.span_events) {
+    if (spanEvent.attributes) {
+      for (const [key, value] of Object.entries(spanEvent.attributes)) {
+        const newValue = convertSpanEventAttributeValues(key, value)
+        if (newValue !== undefined) {
+          spanEvent.attributes[key] = newValue
+        } else {
+          delete spanEvent.attributes[key] // delete from attributes if undefined
+        }
+      }
+      if (Object.entries(spanEvent.attributes).length === 0) {
+        delete spanEvent.attributes
+      }
+    }
+  }
+}
+
+function convertSpanEventAttributeValues (key, value, depth = 0) {
+  if (typeof value === 'string') {
+    return {
+      type: 0,
+      string_value: value
+    }
+  } else if (typeof value === 'boolean') {
+    return {
+      type: 1,
+      bool_value: value
+    }
+  } else if (Number.isInteger(value)) {
+    return {
+      type: 2,
+      int_value: value
+    }
+  } else if (typeof value === 'number') {
+    return {
+      type: 3,
+      double_value: value
+    }
+  } else if (Array.isArray(value)) {
+    if (depth === 0) {
+      const convertedArray = value
+        .map((val) => convertSpanEventAttributeValues(key, val, 1))
+        .filter((convertedVal) => convertedVal !== undefined)
+
+      // Only include array_value if there are valid elements
+      if (convertedArray.length > 0) {
+        return {
+          type: 4,
+          array_value: convertedArray
+        }
+      } else {
+        // If all elements were unsupported, return undefined
+        return undefined
+      }
+    } else {
+      memoizedLogDebug(key, 'Encountered nested array data type for span event v0.4 encoding. ' +
+        `Skipping encoding key: ${key}: with value: ${typeof value}.`
+      )
+      return undefined
+    }
+  } else {
+    memoizedLogDebug(key, 'Encountered unsupported data type for span event v0.4 encoding, key: ' +
+       `${key}: with value: ${typeof value}. Skipping encoding of pair.`
+    )
+    return undefined
+  }
+}
+
 module.exports = { AgentEncoder }

packages/dd-trace/src/encode/0.5.js

@@ -7,7 +7,13 @@ const ARRAY_OF_TWO = 0x92
 const ARRAY_OF_TWELVE = 0x9c
 
 function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span, false))
+  span = normalizeSpan(truncateSpan(span, false))
+  // ensure span events are encoded as tags
+  if (span.span_events) {
+    span.meta.events = JSON.stringify(span.span_events)
+    delete span.span_events
+  }
+  return span
 }
 
 class AgentEncoder extends BaseEncoder {

packages/dd-trace/src/exporters/agent/index.js

@@ -24,7 +24,8 @@ class AgentExporter {
       prioritySampler,
       lookup,
       protocolVersion,
-      headers
+      headers,
+      config
     })
 
     this._timer = undefined

packages/dd-trace/src/exporters/agent/writer.js

@@ -10,15 +10,16 @@ const BaseWriter = require('../common/writer')
 const METRIC_PREFIX = 'datadog.tracer.node.exporter.agent'
 
 class Writer extends BaseWriter {
-  constructor ({ prioritySampler, lookup, protocolVersion, headers }) {
+  constructor ({ prioritySampler, lookup, protocolVersion, headers, config = {} }) {
     super(...arguments)
     const AgentEncoder = getEncoder(protocolVersion)
 
     this._prioritySampler = prioritySampler
     this._lookup = lookup
     this._protocolVersion = protocolVersion
-    this._encoder = new AgentEncoder(this)
     this._headers = headers
+    this._config = config
+    this._encoder = new AgentEncoder(this)
   }
 
   _sendPayload (data, count, done) {