[test optimization] Report code coverage relative to the repository root, not the project's root dir or working directory (#4903)

juan-fernandez · web-flow · commit 920d2a2768b7 · 2024-11-19T17:12:39.000+01:00
diff --git a/integration-tests/ci-visibility/subproject/dependency.js b/integration-tests/ci-visibility/subproject/dependency.js
@@ -0,0 +1,3 @@
+module.exports = function (a, b) {
+  return a + b
+}
diff --git a/integration-tests/ci-visibility/subproject/subproject-test.js b/integration-tests/ci-visibility/subproject/subproject-test.js
@@ -1,9 +1,10 @@
 // eslint-disable-next-line
 const { expect } = require('chai')
+const dependency = require('./dependency')
 
 describe('subproject-test', () => {
   it('can run', () => {
     // eslint-disable-next-line
-    expect(1).to.equal(1)
+    expect(dependency(1, 2)).to.equal(3)
   })
 })
diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js
@@ -275,6 +275,7 @@ versions.forEach(version => {
               }
             )
           })
+
           it('can report code coverage', (done) => {
             const libraryConfigRequestPromise = receiver.payloadReceived(
               ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting')
@@ -355,6 +356,7 @@ versions.forEach(version => {
               done()
             })
           })
+
           it('does not report code coverage if disabled by the API', (done) => {
             receiver.setSettings({
               itr_enabled: false,
@@ -390,6 +392,7 @@ versions.forEach(version => {
               }
             )
           })
+
           it('can skip suites received by the intelligent test runner API and still reports code coverage',
             (done) => {
               receiver.setSuitesToSkip([{
@@ -463,6 +466,7 @@ versions.forEach(version => {
                 }
               )
             })
+
           it('does not skip tests if git metadata upload fails', (done) => {
             receiver.setSuitesToSkip([{
               type: 'suite',
@@ -505,6 +509,7 @@ versions.forEach(version => {
               }
             )
           })
+
           it('does not skip tests if test skipping is disabled by the API', (done) => {
             receiver.setSettings({
               itr_enabled: true,
@@ -543,6 +548,7 @@ versions.forEach(version => {
               }
             )
           })
+
           it('does not skip suites if suite is marked as unskippable', (done) => {
             receiver.setSettings({
               itr_enabled: true,
@@ -611,6 +617,7 @@ versions.forEach(version => {
               }).catch(done)
             })
           })
+
           it('only sets forced to run if suite was going to be skipped by ITR', (done) => {
             receiver.setSettings({
               itr_enabled: true,
@@ -673,6 +680,7 @@ versions.forEach(version => {
               }).catch(done)
             })
           })
+
           it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => {
             receiver.setSuitesToSkip([{
               type: 'suite',
@@ -709,6 +717,7 @@ versions.forEach(version => {
               }).catch(done)
             })
           })
+
           if (!isAgentless) {
             context('if the agent is not event platform proxy compatible', () => {
               it('does not do any intelligent test runner request', (done) => {
@@ -757,6 +766,7 @@ versions.forEach(version => {
               })
             })
           }
+
           it('reports itr_correlation_id in test suites', (done) => {
             const itrCorrelationId = '4321'
             receiver.setItrCorrelationId(itrCorrelationId)
@@ -783,6 +793,45 @@ versions.forEach(version => {
               }).catch(done)
             })
           })
+
+          it('reports code coverage relative to the repository root, not working directory', (done) => {
+            receiver.setSettings({
+              itr_enabled: true,
+              code_coverage: true,
+              tests_skipping: false
+            })
+
+            const codeCoveragesPromise = receiver
+              .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcov'), (payloads) => {
+                const coveredFiles = payloads
+                  .flatMap(({ payload }) => payload)
+                  .flatMap(({ content: { coverages } }) => coverages)
+                  .flatMap(({ files }) => files)
+                  .map(({ filename }) => filename)
+
+                assert.includeMembers(coveredFiles, [
+                  'ci-visibility/subproject/features/support/steps.js',
+                  'ci-visibility/subproject/features/greetings.feature'
+                ])
+              })
+
+            childProcess = exec(
+              '../../node_modules/nyc/bin/nyc.js node ../../node_modules/.bin/cucumber-js features/*.feature',
+              {
+                cwd: `${cwd}/ci-visibility/subproject`,
+                env: {
+                  ...getCiVisAgentlessConfig(receiver.port)
+                },
+                stdio: 'inherit'
+              }
+            )
+
+            childProcess.on('exit', () => {
+              codeCoveragesPromise.then(() => {
+                done()
+              }).catch(done)
+            })
+          })
         })
 
         context('early flake detection', () => {
diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js
@@ -837,6 +837,63 @@ moduleTypes.forEach(({
           }).catch(done)
         })
       })
+
+      it('reports code coverage relative to the repository root, not working directory', (done) => {
+        receiver.setSettings({
+          itr_enabled: false,
+          code_coverage: true,
+          tests_skipping: false
+        })
+        let command
+
+        if (type === 'commonJS') {
+          const commandSuffix = version === '6.7.0'
+            ? '--config-file cypress-config.json --spec "cypress/e2e/*.cy.js"'
+            : ''
+          command = `../../node_modules/.bin/cypress run ${commandSuffix}`
+        } else {
+          command = `node --loader=${hookFile} ../../cypress-esm-config.mjs`
+        }
+
+        const {
+          NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
+          ...restEnvVars
+        } = getCiVisAgentlessConfig(receiver.port)
+
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcov'), (payloads) => {
+            const coveredFiles = payloads
+              .flatMap(({ payload }) => payload)
+              .flatMap(({ content: { coverages } }) => coverages)
+              .flatMap(({ files }) => files)
+              .map(({ filename }) => filename)
+
+            assert.includeMembers(coveredFiles, [
+              'ci-visibility/subproject/src/utils.tsx',
+              'ci-visibility/subproject/src/App.tsx',
+              'ci-visibility/subproject/src/index.tsx',
+              'ci-visibility/subproject/cypress/e2e/spec.cy.js'
+            ])
+          }, 10000)
+
+        childProcess = exec(
+          command,
+          {
+            cwd: `${cwd}/ci-visibility/subproject`,
+            env: {
+              ...restEnvVars,
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+            },
+            stdio: 'inherit'
+          }
+        )
+
+        childProcess.on('exit', () => {
+          eventsPromise.then(() => {
+            done()
+          }).catch(done)
+        })
+      })
     })
 
     it('still reports correct format if there is a plugin incompatibility', (done) => {
diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js
@@ -1469,6 +1469,48 @@ describe('jest CommonJS', () => {
         eventsPromise.then(done).catch(done)
       })
     })
+
+    it('reports code coverage relative to the repository root, not working directory', (done) => {
+      receiver.setSettings({
+        itr_enabled: true,
+        code_coverage: true,
+        tests_skipping: false
+      })
+
+      const codeCoveragesPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcov'), (payloads) => {
+          const coveredFiles = payloads
+            .flatMap(({ payload }) => payload)
+            .flatMap(({ content: { coverages } }) => coverages)
+            .flatMap(({ files }) => files)
+            .map(({ filename }) => filename)
+
+          assert.includeMembers(coveredFiles, [
+            'ci-visibility/subproject/dependency.js',
+            'ci-visibility/subproject/subproject-test.js'
+          ])
+        }, 5000)
+
+      childProcess = exec(
+        'node ./node_modules/jest/bin/jest --config config-jest.js --rootDir ci-visibility/subproject',
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            PROJECTS: JSON.stringify([{
+              testMatch: ['**/subproject-test*']
+            }])
+          },
+          stdio: 'inherit'
+        }
+      )
+
+      childProcess.on('exit', () => {
+        codeCoveragesPromise.then(() => {
+          done()
+        }).catch(done)
+      })
+    })
   })
 
   context('early flake detection', () => {
diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js
@@ -1085,6 +1085,45 @@ describe('mocha CommonJS', function () {
         }).catch(done)
       })
     })
+
+    it('reports code coverage relative to the repository root, not working directory', (done) => {
+      receiver.setSettings({
+        itr_enabled: true,
+        code_coverage: true,
+        tests_skipping: false
+      })
+
+      const codeCoveragesPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcov'), (payloads) => {
+          const coveredFiles = payloads
+            .flatMap(({ payload }) => payload)
+            .flatMap(({ content: { coverages } }) => coverages)
+            .flatMap(({ files }) => files)
+            .map(({ filename }) => filename)
+
+          assert.includeMembers(coveredFiles, [
+            'ci-visibility/subproject/dependency.js',
+            'ci-visibility/subproject/subproject-test.js'
+          ])
+        }, 5000)
+
+      childProcess = exec(
+        '../../node_modules/nyc/bin/nyc.js node ../../node_modules/mocha/bin/mocha subproject-test.js',
+        {
+          cwd: `${cwd}/ci-visibility/subproject`,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port)
+          },
+          stdio: 'inherit'
+        }
+      )
+
+      childProcess.on('exit', () => {
+        codeCoveragesPromise.then(() => {
+          done()
+        }).catch(done)
+      })
+    })
   })
 
   context('early flake detection', () => {
diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js
@@ -127,6 +127,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
 
       if (repositoryRoot) {
         this.testSourceFile = getTestSuitePath(context.testPath, repositoryRoot)
+        this.repositoryRoot = repositoryRoot
       }
 
       this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled
@@ -667,10 +668,13 @@ function jestAdapterWrapper (jestAdapter, jestVersion) {
          * controls whether coverage is reported.
         */
         if (environment.testEnvironmentOptions?._ddTestCodeCoverageEnabled) {
+          const root = environment.repositoryRoot || environment.rootDir
+
           const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__)
-            .map(filename => getTestSuitePath(filename, environment.rootDir))
+            .map(filename => getTestSuitePath(filename, root))
+
           asyncResource.runInAsyncScope(() => {
-            testSuiteCodeCoverageCh.publish({ coverageFiles, testSuite: environment.testSuite })
+            testSuiteCodeCoverageCh.publish({ coverageFiles, testSuite: environment.testSourceFile })
           })
         }
         testSuiteFinishCh.publish({ status, errorMessage })
diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js
@@ -658,10 +658,22 @@ class CypressPlugin {
           log.warn('There is no active test span in dd:afterEach handler')
           return null
         }
-        const { state, error, isRUMActive, testSourceLine, testSuite, testName, isNew, isEfdRetry } = test
+        const {
+          state,
+          error,
+          isRUMActive,
+          testSourceLine,
+          testSuite,
+          testSuiteAbsolutePath,
+          testName,
+          isNew,
+          isEfdRetry
+        } = test
         if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) {
           const coverageFiles = getCoveredFilenamesFromCoverage(coverage)
-          const relativeCoverageFiles = coverageFiles.map(file => getTestSuitePath(file, this.rootDir))
+          const relativeCoverageFiles = [...coverageFiles, testSuiteAbsolutePath].map(
+            file => getTestSuitePath(file, this.repositoryRoot || this.rootDir)
+          )
           if (!relativeCoverageFiles.length) {
             incrementCountMetric(TELEMETRY_CODE_COVERAGE_EMPTY)
           }
diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js
@@ -88,6 +88,7 @@ afterEach(function () {
     const testInfo = {
       testName: currentTest.fullTitle(),
       testSuite: Cypress.mocha.getRootSuite().file,
+      testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute,
       state: currentTest.state,
       error: currentTest.err,
       isNew: currentTest._ddIsNew,
diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js
@@ -85,7 +85,7 @@ class MochaPlugin extends CiPlugin {
       }
 
       const relativeCoverageFiles = [...coverageFiles, suiteFile]
-        .map(filename => getTestSuitePath(filename, this.sourceRoot))
+        .map(filename => getTestSuitePath(filename, this.repositoryRoot || this.sourceRoot))
 
       const { _traceId, _spanId } = testSuiteSpan.context()
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+module.exports = function (a, b) {`
	`2`	`+ return a + b`
	`3`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,7 @@ class MochaPlugin extends CiPlugin {`
`85`	`85`	`}`
`86`	`86`
`87`	`87`	`const relativeCoverageFiles = [...coverageFiles, suiteFile]`
`88`		`- .map(filename => getTestSuitePath(filename, this.sourceRoot))`
	`88`	`+ .map(filename => getTestSuitePath(filename, this.repositoryRoot \|\| this.sourceRoot))`
`89`	`89`
`90`	`90`	`const { _traceId, _spanId } = testSuiteSpan.context()`
`91`	`91`