[MLOB-2333] chore(llmobs): add telemetry for LLMObs span start (#5359)

* add llmobs telemetry span start counts

* change autoinstrumentation to autoinstrumented

* add telemetry to wrap

* integration name instead of kind

Author: sabrenner

packages/dd-trace/src/llmobs/plugins/base.js

@@ -2,6 +2,7 @@
 
 const log = require('../../log')
 const { storage: llmobsStorage } = require('../storage')
+const telemetry = require('../telemetry')
 
 const TracingPlugin = require('../../plugins/tracing')
 const LLMObsTagger = require('../tagger')
@@ -36,6 +37,8 @@ class LLMObsPlugin extends TracingPlugin {
     // register options may not be set for operations we do not trace with llmobs
     // ie OpenAI fine tuning jobs, file jobs, etc.
     if (registerOptions) {
+      telemetry.incrementLLMObsSpanStartCount({ autoinstrumented: true, integration: this.constructor.id })
+
       ctx.llmobs = {} // initialize context-based namespace
       llmobsStorage.enterWith({ span })
       ctx.llmobs.parent = parent

packages/dd-trace/src/llmobs/plugins/bedrockruntime.js

@@ -1,6 +1,7 @@
 const BaseLLMObsPlugin = require('./base')
 const { storage } = require('../../../../datadog-core')
 const llmobsStore = storage('llmobs')
+const telemetry = require('../telemetry')
 
 const {
   extractRequestParams,
@@ -46,6 +47,8 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
   }
 
   setLLMObsTags ({ request, span, response, modelProvider, modelName }) {
+    telemetry.incrementLLMObsSpanStartCount({ autoinstrumented: true, integration: 'bedrock' })
+
     const parent = llmobsStore.getStore()?.span
     this._tagger.registerLLMObsSpan(span, {
       parent,

packages/dd-trace/src/llmobs/plugins/langchain/index.js

@@ -21,6 +21,7 @@ const LlmHandler = require('./handlers/llm')
 const EmbeddingHandler = require('./handlers/embedding')
 
 class LangChainLLMObsPlugin extends LLMObsPlugin {
+  static get id () { return 'langchain' }
   static get prefix () {
     return 'tracing:apm:langchain:invoke'
   }

packages/dd-trace/src/llmobs/plugins/openai.js

@@ -3,6 +3,7 @@
 const LLMObsPlugin = require('./base')
 
 class OpenAiLLMObsPlugin extends LLMObsPlugin {
+  static get id () { return 'openai' }
   static get prefix () {
     return 'tracing:apm:openai:request'
   }

packages/dd-trace/src/llmobs/sdk.js

@@ -14,6 +14,7 @@ const Span = require('../opentracing/span')
 
 const tracerVersion = require('../../../../package.json').version
 const logger = require('../log')
+const telemetry = require('./telemetry')
 
 const LLMObsTagger = require('./tagger')
 
@@ -88,6 +89,8 @@ class LLMObs extends NoopLLMObs {
 
     const kind = validateKind(options.kind) // will throw if kind is undefined or not an expected kind
 
+    telemetry.incrementLLMObsSpanStartCount({ autoinstrumented: false, kind })
+
     // name is required for spans generated with `trace`
     // while `kind` is required, this should never throw (as otherwise it would have thrown above)
     const name = options.name || kind
@@ -133,6 +136,8 @@ class LLMObs extends NoopLLMObs {
     const llmobs = this
 
     function wrapped () {
+      telemetry.incrementLLMObsSpanStartCount({ autoinstrumented: false, kind })
+
       const span = llmobs._tracer.scope().active()
       const fnArgs = arguments
 

packages/dd-trace/src/llmobs/telemetry.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const telemetryMetrics = require('../telemetry/metrics')
+const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs')
+
+function incrementLLMObsSpanStartCount (tags, value = 1) {
+  llmobsMetrics.count('span.start', tags).inc(value)
+}
+
+module.exports = {
+  incrementLLMObsSpanStartCount
+}