Add support for Mocha (no parallel mode)

Author: Mariovido

integration-tests/mocha/mocha.spec.js

@@ -48,7 +48,9 @@ const {
   TEST_MANAGEMENT_IS_DISABLED,
   DD_CAPABILITIES_TEST_IMPACT_ANALYSIS,
   DD_CAPABILITIES_EARLY_FLAKE_DETECTION,
-  DD_CAPABILITIES_AUTO_TEST_RETRIES
+  DD_CAPABILITIES_AUTO_TEST_RETRIES,
+  TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
+  TEST_HAS_FAILED_ALL_RETRIES
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -2566,6 +2568,147 @@ describe('mocha CommonJS', function () {
   })
 
   context('test management', () => {
+    context('attempt to fix', () => {
+      beforeEach(() => {
+        receiver.setTestManagementTests({
+          mocha: {
+            suites: {
+              'ci-visibility/test-management/test-attempt-to-fix-1.js': {
+                tests: {
+                  'attempt to fix tests can attempt to fix a test': {
+                    properties: {
+                      attempt_to_fix: true
+                    }
+                  }
+                }
+              }
+            }
+          }
+        })
+      })
+
+      const getTestAssertions = (isAttemptToFix) =>
+        receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            const testSession = events.find(event => event.type === 'test_session_end').content
+
+            if (isAttemptToFix) {
+              assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
+            } else {
+              assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
+            }
+
+            const resourceNames = tests.map(span => span.resource)
+
+            assert.includeMembers(resourceNames,
+              [
+                'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test'
+              ]
+            )
+
+            const retriedTests = tests.filter(
+              test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test'
+            )
+
+            for (let i = 0; i < retriedTests.length; i++) {
+              const test = retriedTests[i]
+              if (isAttemptToFix && i !== 0) {
+                assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
+                assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
+                assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
+              } else {
+                assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX)
+                assert.notProperty(test.meta, TEST_IS_RETRY)
+                assert.notProperty(test.meta, TEST_RETRY_REASON)
+              }
+
+              if (isAttemptToFix && i === retriedTests.length - 1) {
+                assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true')
+              }
+            }
+          })
+
+      const runAttemptToFixTest = (done, isAttemptToFix, isQuarantined, extraEnvVars = {}) => {
+        let stdout = ''
+        const testAssertionsPromise = getTestAssertions(isAttemptToFix)
+
+        childProcess = exec(
+          runTestsWithCoverageCommand,
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              TESTS_TO_RUN: JSON.stringify([
+                './test-management/test-attempt-to-fix-1.js'
+              ]),
+              SHOULD_CHECK_RESULTS: '1',
+              ...extraEnvVars
+            },
+            stdio: 'inherit'
+          }
+        )
+
+        childProcess.stdout.on('data', (data) => {
+          stdout += data
+        })
+
+        childProcess.on('exit', exitCode => {
+          testAssertionsPromise.then(() => {
+            assert.include(stdout, 'I am running when attempt to fix')
+            if (isAttemptToFix && isQuarantined) {
+              // even though a test fails, the exit code is 0 because the test is quarantined
+              assert.equal(exitCode, 0)
+            } else {
+              assert.equal(exitCode, 1)
+            }
+            done()
+          }).catch(done)
+        })
+      }
+
+      it('can attempt to fix tests', (done) => {
+        receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+        runAttemptToFixTest(done, true, false)
+      })
+
+      it('does not attempt to fix tests if test management is not enabled', (done) => {
+        receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } })
+
+        runAttemptToFixTest(done, false, false)
+      })
+
+      it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+        receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+        runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
+      })
+
+      it('does not fail retry if a test is quarantined', (done) => {
+        receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+        receiver.setTestManagementTests({
+          mocha: {
+            suites: {
+              'ci-visibility/test-management/test-attempt-to-fix-1.js': {
+                tests: {
+                  'attempt to fix tests can attempt to fix a test': {
+                    properties: {
+                      attempt_to_fix: true,
+                      quarantined: true
+                    }
+                  }
+                }
+              }
+            }
+          }
+        })
+
+        runAttemptToFixTest(done, true, true)
+      })
+    })
+
     context('disabled', () => {
       beforeEach(() => {
         receiver.setTestManagementTests({

packages/datadog-instrumentations/src/mocha/main.js

@@ -30,7 +30,9 @@ const {
   newTests,
   testsQuarantined,
   getTestFullName,
-  getRunTestsWrapper
+  getRunTestsWrapper,
+  testsAttemptToFix,
+  testsStatuses
 } = require('./utils')
 
 require('./common')
@@ -138,16 +140,26 @@ function getOnEndHandler (isParallel) {
       }
     }
 
+    // We substract the errors of attempt to fix tests (quarantined or disabled) from the total number of failures
     // We subtract the errors from quarantined tests from the total number of failures
     if (config.isTestManagementTestsEnabled) {
       let numFailedQuarantinedTests = 0
+      let numFailedRetriedQuarantinedOrDisabledTests = 0
+      for (const test of testsAttemptToFix) {
+        const testName = getTestFullName(test, true, false)
+        const testProperties = getTestProperties(test, config.testManagementTests)
+        if (isTestFailed(test) && (testProperties.isQuarantined || testProperties.isDisabled)) {
+          const failedTests = testsStatuses.get(testName).filter(status => status === 'fail')
+          numFailedRetriedQuarantinedOrDisabledTests += failedTests.length
+        }
+      }
       for (const test of testsQuarantined) {
         if (isTestFailed(test)) {
           numFailedQuarantinedTests++
         }
       }
-      this.stats.failures -= numFailedQuarantinedTests
-      this.failures -= numFailedQuarantinedTests
+      this.stats.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests
+      this.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests
     }
 
     if (status === 'fail') {
@@ -193,6 +205,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) {
     if (err) {
       config.testManagementTests = {}
       config.isTestManagementTestsEnabled = false
+      config.testManagementAttemptToFixRetries = 0
     } else {
       config.testManagementTests = receivedTestManagementTests
     }
@@ -260,6 +273,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) {
     config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
     config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
     config.isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
+    config.testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries
     // ITR and auto test retries are not supported in parallel mode yet
     config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled
     config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled
@@ -637,6 +651,7 @@ addHook({
     if (config.isTestManagementTestsEnabled) {
       const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {}
       newWorkerArgs._ddIsTestManagementTestsEnabled = true
+      newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries
       newWorkerArgs._ddTestManagementTests = {
         mocha: {
           suites: {
@@ -661,9 +676,19 @@ addHook({
       .map(event => event.data)
 
     for (const test of tests) {
+      const testProperties = getTestProperties(test, config.testManagementTests)
+      if (config.isTestManagementTestsEnabled) {
+        // `testsAttemptToFix` is filled in the worker process, so we need to use the test results to fill it here too.
+        // `testsQuarantined` is filled in the worker process, so we need to use the test results to fill it here too.
+        if (testProperties.isAttemptToFix) {
+          testsAttemptToFix.add(test)
+        } else if (testProperties.isQuarantined) {
+          testsQuarantined.add(test)
+        }
+      }
       // `newTests` is filled in the worker process, so we need to use the test results to fill it here too.
-      if (config.isKnownTestsEnabled && isNewTest(test, config.knownTests)) {
-        const testFullName = getTestFullName(test)
+      if (config.isKnownTestsEnabled && isNewTest(test, config.knownTests) && !testProperties.isAttemptToFix) {
+        const testFullName = getTestFullName(test, false, true)
         const tests = newTests[testFullName]
 
         if (!tests) {
@@ -672,10 +697,6 @@ addHook({
           tests.push(test)
         }
       }
-      // `testsQuarantined` is filled in the worker process, so we need to use the test results to fill it here too.
-      if (config.isTestManagementTestsEnabled && getTestProperties(test, config.testManagementTests).isQuarantined) {
-        testsQuarantined.add(test)
-      }
     }
     return testFileResult
   })