[test optimization] [SDTEST-1630] Attempt to fix flaky tests implementation

integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature

@@ -0,0 +1,4 @@
+Feature: Attempt to fix
+  Scenario: Say attempt to fix
+    When the greeter says attempt to fix
+    Then I should have heard "attempt to fix"

integration-tests/ci-visibility/features-test-management/support/steps.js

@@ -21,3 +21,10 @@ When('the greeter says disabled', function () {
   // expected to fail if not disabled
   this.whatIHeard = 'disabld'
 })
+
+When('the greeter says attempt to fix', function () {
+  // eslint-disable-next-line no-console
+  console.log('I am running') // just to assert whether this is running
+  // expected to fail
+  this.whatIHeard = 'attempt to fx'
+})

integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js

@@ -0,0 +1,13 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('attempt to fix', () => {
+  test('should attempt to fix failed test', async ({ page }) => {
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello Warld'
+    ])
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js

@@ -0,0 +1,9 @@
+const { expect } = require('chai')
+
+describe('attempt to fix tests', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix') // to check if this is being run
+    expect(1 + 2).to.equal(4)
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js

@@ -0,0 +1,9 @@
+const { expect } = require('chai')
+
+describe('attempt to fix tests 2', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix 2') // to check if this is being run
+    expect(1 + 2).to.equal(3)
+  })
+})

integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs

@@ -0,0 +1,9 @@
+import { describe, test, expect } from 'vitest'
+
+describe('attempt to fix tests', () => {
+  test('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running') // to check if this is being run
+    expect(1 + 2).to.equal(4)
+  })
+})

integration-tests/cucumber/cucumber.spec.js

@@ -49,7 +49,9 @@ const {
   TEST_MANAGEMENT_IS_DISABLED,
   DD_CAPABILITIES_TEST_IMPACT_ANALYSIS,
   DD_CAPABILITIES_EARLY_FLAKE_DETECTION,
-  DD_CAPABILITIES_AUTO_TEST_RETRIES
+  DD_CAPABILITIES_AUTO_TEST_RETRIES,
+  TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
+  TEST_HAS_FAILED_ALL_RETRIES
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 
@@ -2031,6 +2033,139 @@ versions.forEach(version => {
     })
 
     context('test management', () => {
+      context('attempt to fix', () => {
+        beforeEach(() => {
+          receiver.setTestManagementTests({
+            cucumber: {
+              suites: {
+                'ci-visibility/features-test-management/attempt-to-fix.feature': {
+                  tests: {
+                    'Say attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+        })
+
+        const getTestAssertions = (isAttemptToFix) =>
+          receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const tests = events.filter(event => event.type === 'test').map(event => event.content)
+              const testSession = events.find(event => event.type === 'test_session_end').content
+
+              if (isAttemptToFix) {
+                assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
+              } else {
+                assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
+              }
+
+              const retriedTests = tests.filter(
+                test => test.meta[TEST_NAME] === 'Say attempt to fix'
+              )
+
+              for (let i = 0; i < retriedTests.length; i++) {
+                const test = retriedTests[i]
+
+                const testResource = 'ci-visibility/features-test-management/attempt-to-fix.feature.' +
+                  'Say attempt to fix'
+                assert.equal(test.resource, testResource)
+
+                if (isAttemptToFix && i !== 0) {
+                  assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
+                  assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
+                  assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
+                } else {
+                  assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX)
+                  assert.notProperty(test.meta, TEST_IS_RETRY)
+                  assert.notProperty(test.meta, TEST_RETRY_REASON)
+                }
+
+                if (isAttemptToFix && i === retriedTests.length - 1) {
+                  assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true')
+                }
+              }
+            })
+
+        const runTest = (done, isAttemptToFix, isQuarantined, extraEnvVars) => {
+          const testAssertionsPromise = getTestAssertions(isAttemptToFix)
+          let stdout = ''
+
+          childProcess = exec(
+            './node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature',
+            {
+              cwd,
+              env: {
+                ...getCiVisAgentlessConfig(receiver.port),
+                ...extraEnvVars
+              },
+              stdio: 'inherit'
+            }
+          )
+
+          childProcess.stdout.on('data', (data) => {
+            stdout += data.toString()
+          })
+
+          childProcess.on('exit', exitCode => {
+            testAssertionsPromise.then(() => {
+              assert.include(stdout, 'I am running')
+              if (isQuarantined) {
+                assert.equal(exitCode, 0)
+              } else {
+                assert.equal(exitCode, 1)
+              }
+              done()
+            }).catch(done)
+          })
+        }
+
+        it('can attempt to fix tests', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, true, false)
+        })
+
+        it('does not attempt to fix tests if test management is not enabled', (done) => {
+          receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } })
+
+          runTest(done, false, false)
+        })
+
+        it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
+        })
+
+        it('does not fail retry if a test is quarantined', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+          receiver.setTestManagementTests({
+            cucumber: {
+              suites: {
+                'ci-visibility/features-test-management/attempt-to-fix.feature': {
+                  tests: {
+                    'Say attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true,
+                        quarantined: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+
+          runTest(done, true, true)
+        })
+      })
+
       context('disabled', () => {
         beforeEach(() => {
           receiver.setTestManagementTests({

integration-tests/cypress/cypress.spec.js

@@ -44,7 +44,10 @@ const {
   TEST_MANAGEMENT_IS_DISABLED,
   DD_CAPABILITIES_TEST_IMPACT_ANALYSIS,
   DD_CAPABILITIES_EARLY_FLAKE_DETECTION,
-  DD_CAPABILITIES_AUTO_TEST_RETRIES
+  DD_CAPABILITIES_AUTO_TEST_RETRIES,
+  TEST_NAME,
+  TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
+  TEST_HAS_FAILED_ALL_RETRIES
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -1775,6 +1778,120 @@ moduleTypes.forEach(({
     })
 
     context('test management', () => {
+      context('attempt to fix', () => {
+        beforeEach(() => {
+          receiver.setTestManagementTests({
+            cypress: {
+              suites: {
+                'cypress/e2e/attempt-to-fix.js': {
+                  tests: {
+                    'attempt to fix is attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+        })
+
+        const getTestAssertions = (isAttemptToFix) =>
+          receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const tests = events.filter(event => event.type === 'test').map(event => event.content)
+              const testSession = events.find(event => event.type === 'test_session_end').content
+
+              if (isAttemptToFix) {
+                assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
+              } else {
+                assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
+              }
+
+              const resourceNames = tests.map(span => span.resource)
+
+              assert.includeMembers(resourceNames,
+                [
+                  'cypress/e2e/attempt-to-fix.js.attempt to fix is attempt to fix'
+                ]
+              )
+
+              const retriedTests = tests.filter(
+                test => test.meta[TEST_NAME] === 'attempt to fix is attempt to fix'
+              )
+
+              // Events come in reverse order
+              for (let i = retriedTests.length - 1; i >= 0; i--) {
+                const test = retriedTests[i]
+                if (isAttemptToFix && i !== retriedTests.length - 1) {
+                  assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
+                  assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
+                  assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
+                } else {
+                  assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX)
+                  assert.notProperty(test.meta, TEST_IS_RETRY)
+                  assert.notProperty(test.meta, TEST_RETRY_REASON)
+                }
+
+                if (isAttemptToFix && i === 0) {
+                  assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true')
+                }
+              }
+            })
+
+        const runAttemptToFixTest = (done, isAttemptToFix, extraEnvVars) => {
+          const testAssertionsPromise = getTestAssertions(isAttemptToFix)
+
+          const {
+            NODE_OPTIONS,
+            ...restEnvVars
+          } = getCiVisEvpProxyConfig(receiver.port)
+
+          const specToRun = 'cypress/e2e/attempt-to-fix.js'
+
+          childProcess = exec(
+            version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`,
+            {
+              cwd,
+              env: {
+                ...restEnvVars,
+                CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+                SPEC_PATTERN: specToRun,
+                ...extraEnvVars
+              },
+              stdio: 'pipe'
+            }
+          )
+
+          childProcess.on('exit', (exitCode) => {
+            testAssertionsPromise.then(() => {
+              assert.equal(exitCode, 1)
+              done()
+            }).catch(done)
+          })
+        }
+
+        it('can attempt to fix tests', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runAttemptToFixTest(done, true)
+        })
+
+        it('does not attempt to fix tests if test management is not enabled', (done) => {
+          receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } })
+
+          runAttemptToFixTest(done, false)
+        })
+
+        it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runAttemptToFixTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
+        })
+      })
+
       context('disabled', () => {
         beforeEach(() => {
           receiver.setTestManagementTests({

integration-tests/cypress/e2e/attempt-to-fix.js

@@ -0,0 +1,8 @@
+/* eslint-disable */
+describe('attempt to fix', () => {
+  it('is attempt to fix', () => {
+    cy.visit('/')
+      .get('.hello-world')
+      .should('have.text', 'Hello Warld')
+  })
+})