next.js: complete v14.x compatibility (fixing >=14.2.7) (#4916)

- fixs compatibility with Next.js >=v14.2.7 - 14.x
- previously there were 27 test failures
- note that this doesn't address v15.x, I'll do that in a follow up PR
- Next.js 14.2.7 broke compat when the internal headers concept was replaced with a symbol on the request object
- it was further made complicated by us relying on the removal of said internal headers
  - now that they use a symbol they just keep the data around throughout the various stages of the request
  - for that reason I'm using a `WeakSet` to track the two stages of the request
- @see AIDM-339

Author: tlhunter

.github/workflows/appsec.yml

@@ -211,11 +211,12 @@ jobs:
 
   next:
     strategy:
+      fail-fast: false
       matrix:
         version:
           - 18
           - latest
-        range: ['9.5.0', '11.1.4', '13.2.0', '14.2.6']
+        range: ['9.5.0', '11.1.4', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15']
     runs-on: ubuntu-latest
     env:
       PLUGINS: next

.github/workflows/plugins.yml

@@ -739,11 +739,12 @@ jobs:
   # TODO: fix performance issues and test more Node versions
   next:
     strategy:
+      fail-fast: false
       matrix:
         version:
           - 18
           - latest
-        range: ['9.5.0', '11.1.4', '13.2.0', '14.2.6']
+        range: ['9.5.0', '11.1.4', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15']
     runs-on: ubuntu-latest
     env:
       PLUGINS: next

packages/datadog-instrumentations/src/next.js

@@ -14,8 +14,14 @@ const queryParsedChannel = channel('apm:next:query-parsed')
 const requests = new WeakSet()
 const nodeNextRequestsToNextRequests = new WeakMap()
 
+// Next.js <= 14.2.6
 const MIDDLEWARE_HEADER = 'x-middleware-invoke'
 
+// Next.js >= 14.2.7
+const NEXT_REQUEST_META = Symbol.for('NextInternalRequestMeta')
+const META_IS_MIDDLEWARE = 'middlewareInvoke'
+const encounteredMiddleware = new WeakSet()
+
 function wrapHandleRequest (handleRequest) {
   return function (req, res, pathname, query) {
     return instrument(req, res, () => handleRequest.apply(this, arguments))
@@ -111,6 +117,11 @@ function getPageFromPath (page, dynamicRoutes = []) {
   return getPagePath(page)
 }
 
+function getRequestMeta (req, key) {
+  const meta = req[NEXT_REQUEST_META] || {}
+  return typeof key === 'string' ? meta[key] : meta
+}
+
 function instrument (req, res, error, handler) {
   if (typeof error === 'function') {
     handler = error
@@ -121,8 +132,9 @@ function instrument (req, res, error, handler) {
   res = res.originalResponse || res
 
   // TODO support middleware properly in the future?
-  const isMiddleware = req.headers[MIDDLEWARE_HEADER]
-  if (isMiddleware || requests.has(req)) {
+  const isMiddleware = req.headers[MIDDLEWARE_HEADER] || getRequestMeta(req, META_IS_MIDDLEWARE)
+  if ((isMiddleware && !encounteredMiddleware.has(req)) || requests.has(req)) {
+    encounteredMiddleware.add(req)
     if (error) {
       errorChannel.publish({ error })
     }
@@ -188,7 +200,7 @@ function finish (ctx, result, err) {
 // however, it is not provided as a class function or exported property
 addHook({
   name: 'next',
-  versions: ['>=13.3.0 <14.2.7'],
+  versions: ['>=13.3.0 <15'],
   file: 'dist/server/web/spec-extension/adapters/next-request.js'
 }, NextRequestAdapter => {
   shimmer.wrap(NextRequestAdapter.NextRequestAdapter, 'fromNodeNextRequest', fromNodeNextRequest => {
@@ -203,7 +215,7 @@ addHook({
 
 addHook({
   name: 'next',
-  versions: ['>=11.1 <14.2.7'],
+  versions: ['>=11.1 <15'],
   file: 'dist/server/serve-static.js'
 }, serveStatic => shimmer.wrap(serveStatic, 'serveStatic', wrapServeStatic))
 
@@ -213,7 +225,7 @@ addHook({
   file: 'dist/next-server/server/serve-static.js'
 }, serveStatic => shimmer.wrap(serveStatic, 'serveStatic', wrapServeStatic))
 
-addHook({ name: 'next', versions: ['>=11.1 <14.2.7'], file: 'dist/server/next-server.js' }, nextServer => {
+addHook({ name: 'next', versions: ['>=11.1 <15'], file: 'dist/server/next-server.js' }, nextServer => {
   const Server = nextServer.default
 
   shimmer.wrap(Server.prototype, 'handleRequest', wrapHandleRequest)
@@ -230,7 +242,7 @@ addHook({ name: 'next', versions: ['>=11.1 <14.2.7'], file: 'dist/server/next-se
 })
 
 // `handleApiRequest` changes parameters/implementation at 13.2.0
-addHook({ name: 'next', versions: ['>=13.2 <14.2.7'], file: 'dist/server/next-server.js' }, nextServer => {
+addHook({ name: 'next', versions: ['>=13.2 <15'], file: 'dist/server/next-server.js' }, nextServer => {
   const Server = nextServer.default
   shimmer.wrap(Server.prototype, 'handleApiRequest', wrapHandleApiRequestWithMatch)
   return nextServer
@@ -264,7 +276,7 @@ addHook({
 
 addHook({
   name: 'next',
-  versions: ['>=13 <14.2.7'],
+  versions: ['>=13 <15'],
   file: 'dist/server/web/spec-extension/request.js'
 }, request => {
   const nextUrlDescriptor = Object.getOwnPropertyDescriptor(request.NextRequest.prototype, 'nextUrl')