v5.43.0 proposal

integration-tests/opentelemetry.spec.js

@@ -60,7 +60,7 @@ describe('opentelemetry', () => {
       '@opentelemetry/api@1.8.0',
       '@opentelemetry/instrumentation',
       '@opentelemetry/instrumentation-http',
-      '@opentelemetry/instrumentation-express',
+      '@opentelemetry/instrumentation-express@0.47.1',
       'express'
     ]
     if (satisfies(process.version.slice(1), '>=14')) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.42.0",
+  "version": "5.43.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -89,7 +89,7 @@
     "@datadog/native-iast-rewriter": "2.8.0",
     "@datadog/native-iast-taint-tracking": "3.3.0",
     "@datadog/native-metrics": "^3.1.0",
-    "@datadog/pprof": "5.5.1",
+    "@datadog/pprof": "5.6.0",
     "@datadog/sketches-js": "^2.1.0",
     "@isaacs/ttlcache": "^1.4.1",
     "@opentelemetry/api": ">=1.0.0 <1.9.0",

packages/datadog-instrumentations/src/vitest.js

@@ -94,6 +94,10 @@ function isReporterPackageNewest (vitestPackage) {
   return vitestPackage.h?.name === 'BaseSequencer'
 }
 
+function isBaseSequencer (vitestPackage) {
+  return vitestPackage.b?.name === 'BaseSequencer'
+}
+
 function getChannelPromise (channelToPublishTo) {
   return new Promise(resolve => {
     sessionAsyncResource.runInAsyncScope(() => {
@@ -615,11 +619,22 @@ addHook({
 
 addHook({
   name: 'vitest',
-  versions: ['>=3.0.0'],
+  versions: ['>=3.0.9'],
+  filePattern: 'dist/chunks/coverage.*'
+}, (coveragePackage) => {
+  if (isBaseSequencer(coveragePackage)) {
+    shimmer.wrap(coveragePackage.b.prototype, 'sort', getSortWrapper)
+  }
+  return coveragePackage
+})
+
+addHook({
+  name: 'vitest',
+  versions: ['>=3.0.0 <3.0.9'],
   filePattern: 'dist/chunks/resolveConfig.*'
-}, (randomSequencerPackage) => {
-  shimmer.wrap(randomSequencerPackage.B.prototype, 'sort', getSortWrapper)
-  return randomSequencerPackage
+}, (resolveConfigPackage) => {
+  shimmer.wrap(resolveConfigPackage.B.prototype, 'sort', getSortWrapper)
+  return resolveConfigPackage
 })
 
 // Can't specify file because compiled vitest includes hashes in their files

packages/dd-trace/src/appsec/telemetry/common.js

@@ -7,7 +7,8 @@ const tags = {
   RULE_TRIGGERED: 'rule_triggered',
   WAF_TIMEOUT: 'waf_timeout',
   WAF_VERSION: 'waf_version',
-  EVENT_RULES_VERSION: 'event_rules_version'
+  EVENT_RULES_VERSION: 'event_rules_version',
+  INPUT_TRUNCATED: 'input_truncated'
 }
 
 function getVersionsTags (wafVersion, rulesVersion) {

packages/dd-trace/src/appsec/telemetry/waf.js

@@ -7,6 +7,12 @@ const appsecMetrics = telemetryMetrics.manager.namespace('appsec')
 
 const DD_TELEMETRY_WAF_RESULT_TAGS = Symbol('_dd.appsec.telemetry.waf.result.tags')
 
+const TRUNCATION_FLAGS = {
+  STRING: 1,
+  CONTAINER_SIZE: 2,
+  CONTAINER_DEPTH: 4
+}
+
 function addWafRequestMetrics (store, { duration, durationExt, wafTimeout, errorCode }) {
   store[DD_TELEMETRY_REQUEST_METRICS].duration += duration || 0
   store[DD_TELEMETRY_REQUEST_METRICS].durationExt += durationExt || 0
@@ -58,6 +64,12 @@ function trackWafMetrics (store, metrics) {
     metricTags[tags.WAF_TIMEOUT] = true
   }
 
+  const truncationReason = getTruncationReason(metrics)
+  if (truncationReason > 0) {
+    metricTags[tags.INPUT_TRUNCATED] = true
+    incrementTruncatedMetrics(metrics, truncationReason)
+  }
+
   return metricTags
 }
 
@@ -69,6 +81,7 @@ function getOrCreateMetricTags (store, versionsTags) {
       [tags.REQUEST_BLOCKED]: false,
       [tags.RULE_TRIGGERED]: false,
       [tags.WAF_TIMEOUT]: false,
+      [tags.INPUT_TRUNCATED]: false,
 
       ...versionsTags
     }
@@ -98,6 +111,39 @@ function incrementWafRequests (store) {
   }
 }
 
+function incrementTruncatedMetrics (metrics, truncationReason) {
+  const truncationTags = { truncation_reason: truncationReason }
+  appsecMetrics.count('waf.input_truncated', truncationTags).inc(1)
+
+  if (metrics?.maxTruncatedString) {
+    appsecMetrics.distribution('waf.truncated_value_size', {
+      truncation_reason: TRUNCATION_FLAGS.STRING
+    }).track(metrics.maxTruncatedString)
+  }
+
+  if (metrics?.maxTruncatedContainerSize) {
+    appsecMetrics.distribution('waf.truncated_value_size', {
+      truncation_reason: TRUNCATION_FLAGS.CONTAINER_SIZE
+    }).track(metrics.maxTruncatedContainerSize)
+  }
+
+  if (metrics?.maxTruncatedContainerDepth) {
+    appsecMetrics.distribution('waf.truncated_value_size', {
+      truncation_reason: TRUNCATION_FLAGS.CONTAINER_DEPTH
+    }).track(metrics.maxTruncatedContainerDepth)
+  }
+}
+
+function getTruncationReason ({ maxTruncatedString, maxTruncatedContainerSize, maxTruncatedContainerDepth }) {
+  let reason = 0
+
+  if (maxTruncatedString) reason |= TRUNCATION_FLAGS.STRING
+  if (maxTruncatedContainerSize) reason |= TRUNCATION_FLAGS.CONTAINER_SIZE
+  if (maxTruncatedContainerDepth) reason |= TRUNCATION_FLAGS.CONTAINER_DEPTH
+
+  return reason
+}
+
 module.exports = {
   addWafRequestMetrics,
   trackWafMetrics,

packages/dd-trace/src/config.js

@@ -595,6 +595,7 @@ class Config {
     this._setValue(defaults, 'vertexai.spanCharLimit', 128)
     this._setValue(defaults, 'vertexai.spanPromptCompletionSampleRate', 1.0)
     this._setValue(defaults, 'trace.aws.addSpanPointers', true)
+    this._setValue(defaults, 'trace.nativeSpanEvents', false)
   }
 
   _applyLocalStableConfig () {
@@ -765,6 +766,7 @@ class Config {
       DD_VERTEXAI_SPAN_PROMPT_COMPLETION_SAMPLE_RATE,
       DD_VERTEXAI_SPAN_CHAR_LIMIT,
       DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED,
+      DD_TRACE_NATIVE_SPAN_EVENTS,
       OTEL_METRICS_EXPORTER,
       OTEL_PROPAGATORS,
       OTEL_RESOURCE_ATTRIBUTES,
@@ -977,6 +979,7 @@ class Config {
     this._setBoolean(env, 'trace.aws.addSpanPointers', DD_TRACE_AWS_ADD_SPAN_POINTERS)
     this._setString(env, 'trace.dynamoDb.tablePrimaryKeys', DD_TRACE_DYNAMODB_TABLE_PRIMARY_KEYS)
     this._setArray(env, 'graphqlErrorExtensions', DD_TRACE_GRAPHQL_ERROR_EXTENSIONS)
+    this._setBoolean(env, 'trace.nativeSpanEvents', DD_TRACE_NATIVE_SPAN_EVENTS)
     this._setValue(
       env,
       'vertexai.spanPromptCompletionSampleRate',
@@ -1114,6 +1117,7 @@ class Config {
     this._setString(opts, 'version', options.version || tags.version)
     this._setBoolean(opts, 'inferredProxyServicesEnabled', options.inferredProxyServicesEnabled)
     this._setBoolean(opts, 'graphqlErrorExtensions', options.graphqlErrorExtensions)
+    this._setBoolean(opts, 'trace.nativeSpanEvents', options.trace?.nativeSpanEvents)
 
     // For LLMObs, we want the environment variable to take precedence over the options.
     // This is reliant on environment config being set before options.

packages/dd-trace/src/dogstatsd.js

@@ -234,7 +234,7 @@ class MetricsAggregationClient {
     this._histograms[name].get(tag).record(value)
   }
 
-  count (name, count, tag, monotonic = false) {
+  count (name, count, tag, monotonic = true) {
     if (typeof tag === 'boolean') {
       monotonic = tag
       tag = undefined
@@ -254,8 +254,8 @@ class MetricsAggregationClient {
     this._gauges[name].set(tag, value)
   }
 
-  increment (name, count = 1, tag, monotonic) {
-    this.count(name, count, tag, monotonic)
+  increment (name, count = 1, tag) {
+    this.count(name, count, tag)
   }
 
   decrement (name, count = 1, tag) {

packages/dd-trace/src/encode/0.4.js

@@ -5,11 +5,22 @@ const { Chunk, MsgpackEncoder } = require('../msgpack')
 const log = require('../log')
 const { isTrue } = require('../util')
 const coalesce = require('koalas')
+const { memoize } = require('../log/utils')
 
 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB
 
-function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span, false))
+function formatSpan (span, config) {
+  span = normalizeSpan(truncateSpan(span, false))
+  if (span.span_events) {
+    // ensure span events are encoded as tags if agent doesn't support native top level span events
+    if (!config?.trace?.nativeSpanEvents) {
+      span.meta.events = JSON.stringify(span.span_events)
+      delete span.span_events
+    } else {
+      formatSpanEvents(span)
+    }
+  }
+  return span
 }
 
 class AgentEncoder {
@@ -24,6 +35,7 @@ class AgentEncoder {
       process.env.DD_TRACE_ENCODING_DEBUG,
       false
     ))
+    this._config = this._writer?._config
   }
 
   count () {
@@ -74,16 +86,18 @@ class AgentEncoder {
     this._encodeArrayPrefix(bytes, trace)
 
     for (let span of trace) {
-      span = formatSpan(span)
+      span = formatSpan(span, this._config)
       bytes.reserve(1)
 
-      if (span.type && span.meta_struct) {
-        bytes.buffer[bytes.length - 1] = 0x8d
-      } else if (span.type || span.meta_struct) {
-        bytes.buffer[bytes.length - 1] = 0x8c
-      } else {
-        bytes.buffer[bytes.length - 1] = 0x8b
-      }
+      // this is the original size of the fixed map for span attributes that always exist
+      let mapSize = 11
+
+      // increment the payload map size depending on if some optional fields exist
+      if (span.type) mapSize += 1
+      if (span.meta_struct) mapSize += 1
+      if (span.span_events) mapSize += 1
+
+      bytes.buffer[bytes.length - 1] = 0x80 + mapSize
 
       if (span.type) {
         this._encodeString(bytes, 'type')
@@ -112,6 +126,10 @@ class AgentEncoder {
       this._encodeMap(bytes, span.meta)
       this._encodeString(bytes, 'metrics')
       this._encodeMap(bytes, span.metrics)
+      if (span.span_events) {
+        this._encodeString(bytes, 'span_events')
+        this._encodeObjectAsArray(bytes, span.span_events, new Set())
+      }
       if (span.meta_struct) {
         this._encodeString(bytes, 'meta_struct')
         this._encodeMetaStruct(bytes, span.meta_struct)
@@ -200,6 +218,9 @@ class AgentEncoder {
       case 'number':
         this._encodeFloat(bytes, value)
         break
+      case 'boolean':
+        this._encodeBool(bytes, value)
+        break
       default:
         // should not happen
     }
@@ -258,7 +279,7 @@ class AgentEncoder {
       this._encodeObjectAsArray(bytes, value, circularReferencesDetector)
     } else if (value !== null && typeof value === 'object') {
       this._encodeObjectAsMap(bytes, value, circularReferencesDetector)
-    } else if (typeof value === 'string' || typeof value === 'number') {
+    } else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
       this._encodeValue(bytes, value)
     }
   }
@@ -268,7 +289,7 @@ class AgentEncoder {
     const validKeys = keys.filter(key => {
       const v = value[key]
       return typeof v === 'string' ||
-        typeof v === 'number' ||
+        typeof v === 'number' || typeof v === 'boolean' ||
         (v !== null && typeof v === 'object' && !circularReferencesDetector.has(v))
     })
 
@@ -319,4 +340,79 @@ class AgentEncoder {
   }
 }
 
+const memoizedLogDebug = memoize((key, message) => {
+  log.debug(message)
+  // return something to store in memoize cache
+  return true
+})
+
+function formatSpanEvents (span) {
+  for (const spanEvent of span.span_events) {
+    if (spanEvent.attributes) {
+      for (const [key, value] of Object.entries(spanEvent.attributes)) {
+        const newValue = convertSpanEventAttributeValues(key, value)
+        if (newValue !== undefined) {
+          spanEvent.attributes[key] = newValue
+        } else {
+          delete spanEvent.attributes[key] // delete from attributes if undefined
+        }
+      }
+      if (Object.entries(spanEvent.attributes).length === 0) {
+        delete spanEvent.attributes
+      }
+    }
+  }
+}
+
+function convertSpanEventAttributeValues (key, value, depth = 0) {
+  if (typeof value === 'string') {
+    return {
+      type: 0,
+      string_value: value
+    }
+  } else if (typeof value === 'boolean') {
+    return {
+      type: 1,
+      bool_value: value
+    }
+  } else if (Number.isInteger(value)) {
+    return {
+      type: 2,
+      int_value: value
+    }
+  } else if (typeof value === 'number') {
+    return {
+      type: 3,
+      double_value: value
+    }
+  } else if (Array.isArray(value)) {
+    if (depth === 0) {
+      const convertedArray = value
+        .map((val) => convertSpanEventAttributeValues(key, val, 1))
+        .filter((convertedVal) => convertedVal !== undefined)
+
+      // Only include array_value if there are valid elements
+      if (convertedArray.length > 0) {
+        return {
+          type: 4,
+          array_value: convertedArray
+        }
+      } else {
+        // If all elements were unsupported, return undefined
+        return undefined
+      }
+    } else {
+      memoizedLogDebug(key, 'Encountered nested array data type for span event v0.4 encoding. ' +
+        `Skipping encoding key: ${key}: with value: ${typeof value}.`
+      )
+      return undefined
+    }
+  } else {
+    memoizedLogDebug(key, 'Encountered unsupported data type for span event v0.4 encoding, key: ' +
+       `${key}: with value: ${typeof value}. Skipping encoding of pair.`
+    )
+    return undefined
+  }
+}
+
 module.exports = { AgentEncoder }

packages/dd-trace/src/encode/0.5.js

@@ -7,7 +7,13 @@ const ARRAY_OF_TWO = 0x92
 const ARRAY_OF_TWELVE = 0x9c
 
 function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span, false))
+  span = normalizeSpan(truncateSpan(span, false))
+  // ensure span events are encoded as tags
+  if (span.span_events) {
+    span.meta.events = JSON.stringify(span.span_events)
+    delete span.span_events
+  }
+  return span
 }
 
 class AgentEncoder extends BaseEncoder {