v3.17.0

.github/actions/check-release-branch/action.yml

@@ -0,0 +1,29 @@
+name: Check release branch
+inputs:
+  release-branch:
+    description: 'Release branch to check'
+    required: true
+    default: '3.x'
+runs:
+  using: composite
+  steps:
+    - name: Squash current branch commits
+      id: get-commit-to-cherry-pick
+      shell: bash
+      run: |
+        git checkout -b ${{github.head_ref}}-squashed
+        git reset --soft refs/remotes/origin/master
+        git add -A
+        git config user.email "dummy@commit.com"
+        git config user.name "Dummy commit"
+        git commit -m "squashed commit"
+        echo "commit-to-cherry-pick=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+    - name: Attempt merge current branch to ${{ inputs.release-branch }}
+      shell: bash
+      run: |
+        git checkout ${{ inputs.release-branch }}
+        {
+          git cherry-pick ${{ steps.get-commit-to-cherry-pick.outputs.commit-to-cherry-pick}}
+        } || {
+          git status ; git diff ; exit 1
+        }

.github/workflows/project.yml

@@ -14,6 +14,9 @@ concurrency:
 jobs:
   integration:
     strategy:
+      # when one version fails, say 14, all the other versions are stopped
+      # setting fail-fast to false in an attempt to prevent this from happening
+      fail-fast: false
       matrix:
         version: [14, 16, 18, latest]
     runs-on: ubuntu-latest
@@ -28,6 +31,21 @@ jobs:
       - run: sudo sysctl -w kernel.core_pattern='|/bin/false'
       - run: yarn test:integration
 
+  integration-ci:
+    strategy:
+      matrix:
+        version: [14, latest]
+        framework: [cucumber, cypress, playwright]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/node/setup
+      - run: yarn install
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.version }}
+      - run: yarn test:integration:${{ matrix.framework }}
+
   lint:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release-branches-check.yml

@@ -0,0 +1,28 @@
+name: Release branches check
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  check-release-branch-v2:
+    if: "!contains(github.event.pull_request.labels.*.name, 'dont-land-on-v2.x')"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/check-release-branch
+        with:
+          release-branch: v2.x
+
+  check-release-branch-v3:
+    if: "!contains(github.event.pull_request.labels.*.name, 'dont-land-on-v3.x')"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/check-release-branch
+        with:
+          release-branch: v3.x

LICENSE-3rdparty.csv

@@ -36,6 +36,7 @@ dev,chalk,MIT,Copyright Sindre Sorhus
 dev,checksum,MIT,Copyright Daniel D. Shaw
 dev,cli-table3,MIT,Copyright 2014 James Talmage
 dev,dotenv,BSD-2-Clause,Copyright 2015 Scott Motte
+dev,esbuild,MIT,Copyright (c) 2020 Evan Wallace
 dev,eslint,MIT,Copyright JS Foundation and other contributors https://js.foundation
 dev,eslint-config-standard,MIT,Copyright Feross Aboukhadijeh
 dev,eslint-plugin-import,MIT,Copyright 2015 Ben Mosher

README.md

@@ -151,6 +151,19 @@ $ yarn lint
 ```
 
 
+### Experimental ESM Support
+
+ESM support is currently in the experimental stages, while CJS has been supported
+since inception. This means that code loaded using `require()` should work fine
+but code loaded using `import` might not always work.
+
+Use the following command to enable experimental ESM support with your application:
+
+```sh
+node --loader dd-trace/loader-hook.mjs entrypoint.js
+```
+
+
 ### Benchmarks
 
 Our microbenchmarks live in `benchmark/sirun`. Each directory in there
@@ -175,6 +188,42 @@ That said, even if your application runs on Lambda, any core instrumentation iss
 Regardless of where you open the issue, someone at Datadog will try to help.
 
 
+## Bundling
+
+Generally, `dd-trace` works by intercepting `require()` calls that a Node.js application makes when loading modules. This includes modules that are built-in to Node.js, like the `fs` module for accessing the filesystem, as well as modules installed from the npm registry, like the `pg` database module.
+
+Also generally, bundlers work by crawling all of the `require()` calls that an application makes to files on disk, replacing the `require()` calls with custom code, and then concatenating all of the resulting JavaScript into one "bundled" file. When a built-in module is loaded, like `require('fs')`, that call can then remain the same in the resulting bundle.
+
+Fundamentally APM tools like `dd-trace` stop working at this point. Perhaps they continue to intercept the calls for built-in modules but don't intercept calls to third party libraries. This means that by default when you bundle a `dd-trace` app with a bundler it is likely to capture information about disk access (via `fs`) and outbound HTTP requests (via `http`), but will otherwise omit calls to third party libraries (like extracting incoming request route information for the `express` framework or showing which query is run for the `mysql` database client).
+
+To get around this, one can treat all third party modules, or at least third party modules that the APM needs to instrument, as being "external" to the bundler. With this setting the instrumented modules remain on disk and continue to be loaded via `require()` while the non-instrumented modules are bundled. Sadly this results in a build with many extraneous files and starts to defeat the purpose of bundling. 
+
+For these reasons it's necessary to have custom-built bundler plugins. Such plugins are able to instruct the bundler on how to behave, injecting intermediary code and otherwise intercepting the "translated" `require()` calls. The result is that many more packages are then included in the bundled JavaScript file. Some applications can have 100% of modules bundled, however native modules still need to remain external to the bundle.
+
+### Esbuild Support
+
+This library provides experimental esbuild support in the form of an esbuild plugin, and currently requires at least Node.js v16.17 or v18.7. To use the plugin, make sure you have `dd-trace@3+` installed, and then require the `dd-trace/esbuild` module when building your bundle.
+
+Here's an example of how one might use `dd-trace` with esbuild:
+
+```javascript
+const ddPlugin = require('dd-trace/esbuild')
+const esbuild = require('esbuild')
+
+esbuild.build({
+  entryPoints: ['app.js'], 
+  bundle: true,
+  outfile: 'out.js',
+  plugins: [ddPlugin],
+  platform: 'node', // allows built-in modules to be required
+  target: ['node16']
+}).catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
+```
+
+
 ## Security Vulnerabilities
 
 If you have found a security issue, please contact the security team directly at [security@datadoghq.com](mailto:security@datadoghq.com).

esbuild.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('./packages/datadog-esbuild/index.js')

integration-tests/ci-visibility-intake.js

@@ -169,6 +169,39 @@ class FakeCiVisIntake extends FakeAgent {
     infoResponse = DEFAULT_INFO_RESPONSE
   }
 
+  // Similar to gatherPayloads but resolves if enough payloads have been gathered
+  // to make the assertions pass. It times out after maxGatheringTime so it should
+  // always be faster or as fast as gatherPayloads
+  gatherPayloadsMaxTimeout (payloadMatch, onPayload, maxGatheringTime = 15000) {
+    const payloads = []
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        try {
+          onPayload(payloads)
+          resolve()
+        } catch (e) {
+          reject(e)
+        } finally {
+          this.off('message', messageHandler)
+        }
+      }, maxGatheringTime)
+      const messageHandler = (message) => {
+        if (!payloadMatch || payloadMatch(message)) {
+          payloads.push(message)
+          try {
+            onPayload(payloads)
+            clearTimeout(timeoutId)
+            this.off('message', messageHandler)
+            resolve()
+          } catch (e) {
+            // we'll try again when a new payload arrives
+          }
+        }
+      }
+      this.on('message', messageHandler)
+    })
+  }
+
   gatherPayloads (payloadMatch, gatheringTime = 15000) {
     const payloads = []
     return new Promise((resolve, reject) => {

integration-tests/cucumber.spec.js → integration-tests/cucumber/cucumber.spec.js

@@ -10,8 +10,8 @@ const {
   createSandbox,
   getCiVisAgentlessConfig,
   getCiVisEvpProxyConfig
-} = require('./helpers')
-const { FakeCiVisIntake } = require('./ci-visibility-intake')
+} = require('../helpers')
+const { FakeCiVisIntake } = require('../ci-visibility-intake')
 const {
   TEST_STATUS,
   TEST_COMMAND,
@@ -23,7 +23,7 @@ const {
   TEST_MODULE_ITR_SKIPPING_ENABLED,
   TEST_ITR_TESTS_SKIPPED,
   TEST_CODE_COVERAGE_LINES_TOTAL
-} = require('../packages/dd-trace/src/plugins/util/test')
+} = require('../../packages/dd-trace/src/plugins/util/test')
 
 const isOldNode = semver.satisfies(process.version, '<=12')
 const versions = ['7.0.0', isOldNode ? '8' : 'latest']
@@ -63,7 +63,7 @@ versions.forEach(version => {
           envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
         })
         it('can run and report tests', (done) => {
-          receiver.gatherPayloads(({ url }) => url.endsWith('/api/v2/citestcycle'), 5000).then((payloads) => {
+          receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
             const events = payloads.flatMap(({ payload }) => payload.events)
 
             const testSessionEvent = events.find(event => event.type === 'test_session_end')
@@ -151,9 +151,7 @@ versions.forEach(version => {
               assert.equal(stepEvent.content.name, 'cucumber.step')
               assert.property(stepEvent.content.meta, 'cucumber.step')
             })
-
-            done()
-          }).catch(done)
+          }, 5000).then(() => done()).catch(done)
 
           childProcess = exec(
             runTestsCommand,

integration-tests/cypress.spec.js → integration-tests/cypress/cypress.spec.js

@@ -10,16 +10,16 @@ const {
   createSandbox,
   getCiVisAgentlessConfig,
   getCiVisEvpProxyConfig
-} = require('./helpers')
-const { FakeCiVisIntake } = require('./ci-visibility-intake')
-const webAppServer = require('./ci-visibility/web-app-server')
+} = require('../helpers')
+const { FakeCiVisIntake } = require('../ci-visibility-intake')
+const webAppServer = require('../ci-visibility/web-app-server')
 const {
   TEST_STATUS,
   TEST_COMMAND,
   TEST_MODULE,
   TEST_FRAMEWORK_VERSION,
   TEST_TOOLCHAIN
-} = require('../packages/dd-trace/src/plugins/util/test')
+} = require('../../packages/dd-trace/src/plugins/util/test')
 
 // TODO: remove when 2.x support is removed.
 // This is done because from playwright@>=1.22.0 node 12 is not supported
@@ -29,7 +29,7 @@ const versions = ['6.7.0', isOldNode ? '11.2.0' : 'latest']
 versions.forEach((version) => {
   describe(`cypress@${version}`, function () {
     this.retries(2)
-    this.timeout(45000)
+    this.timeout(60000)
     let sandbox, cwd, receiver, childProcess, webAppPort
     before(async () => {
       sandbox = await createSandbox([`cypress@${version}`], true)
@@ -61,7 +61,7 @@ versions.forEach((version) => {
             ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
           const reportUrl = reportMethod === 'agentless' ? '/api/v2/citestcycle' : '/evp_proxy/v2/api/v2/citestcycle'
 
-          receiver.gatherPayloads(({ url }) => url === reportUrl, 25000).then((payloads) => {
+          receiver.gatherPayloadsMaxTimeout(({ url }) => url === reportUrl, payloads => {
             const events = payloads.flatMap(({ payload }) => payload.events)
 
             const testSessionEvent = events.find(event => event.type === 'test_session_end')
@@ -141,9 +141,7 @@ versions.forEach((version) => {
               assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
               assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
             })
-
-            done()
-          }).catch(done)
+          }, 25000).then(() => done()).catch(done)
 
           const {
             NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
@@ -160,7 +158,7 @@ versions.forEach((version) => {
                 ...restEnvVars,
                 CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
               },
-              stdio: 'on'
+              stdio: 'pipe'
             }
           )
         })

integration-tests/esbuild.spec.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+'use strict'
+
+const chproc = require('child_process')
+const path = require('path')
+
+const CWD = process.cwd()
+const TEST_DIR = path.join(__dirname, 'esbuild')
+
+// eslint-disable-next-line no-console
+console.log(`cd ${TEST_DIR}`)
+process.chdir(TEST_DIR)
+
+// eslint-disable-next-line no-console
+console.log('npm run build')
+chproc.execSync('npm run build')
+
+// eslint-disable-next-line no-console
+console.log('npm run built')
+try {
+  chproc.execSync('npm run built', {
+    timeout: 1000 * 30
+  })
+} catch (err) {
+  // eslint-disable-next-line no-console
+  console.error(err)
+  process.exit(1)
+} finally {
+  process.chdir(CWD)
+}

integration-tests/esbuild/.gitignore

@@ -0,0 +1 @@
+out.js

integration-tests/esbuild/basic-test.js

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+
+// TODO: add support for Node.js v14.17+ and v16.0+
+if (Number(process.versions.node.split('.')[0]) < 16) {
+  console.error(`Skip esbuild test for node@${process.version}`) // eslint-disable-line no-console
+  process.exit(0)
+}
+
+const tracer = require('../../').init() // dd-trace
+
+const assert = require('assert')
+const express = require('express')
+const http = require('http')
+
+const app = express()
+const PORT = 31415
+
+assert.equal(express.static.mime.types.ogg, 'audio/ogg')
+
+const server = app.listen(PORT, () => {
+  setImmediate(() => {
+    http.request(`http://localhost:${PORT}`).end() // query to self
+  })
+})
+
+app.get('/', async (_req, res) => {
+  assert.equal(
+    tracer.scope().active().context()._tags.component,
+    'express',
+    `the sample app bundled by esbuild is not properly instrumented. using node@${process.version}`
+  ) // bad exit
+
+  res.json({ narwhal: 'bacons' })
+
+  setImmediate(() => {
+    server.close() // clean exit
+    setImmediate(() => {
+      process.exit(0)
+    })
+  })
+})

integration-tests/esbuild/build.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const ddPlugin = require('../../esbuild') // dd-trace/esbuild
+const esbuild = require('esbuild')
+
+esbuild.build({
+  entryPoints: ['basic-test.js'],
+  bundle: true,
+  outfile: 'out.js',
+  plugins: [ddPlugin],
+  platform: 'node',
+  target: ['node16']
+}).catch((err) => {
+  console.error(err) // eslint-disable-line no-console
+  process.exit(1)
+})