From b51113a387558fffef9228a647f89898e18f2581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Mon, 17 Mar 2025 11:27:29 +0100 Subject: [PATCH 01/16] Add Attempt to Fix num of retries variable --- .../src/ci-visibility/exporters/ci-visibility-exporter.js | 6 ++++-- .../src/ci-visibility/requests/get-library-configuration.js | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index a87078ed7e7..b615f862d2b 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -215,7 +215,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isFlakyTestRetriesEnabled, isDiEnabled, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } = remoteConfiguration return { isCodeCoverageEnabled, @@ -229,7 +230,8 @@ class CiVisibilityExporter extends AgentInfoExporter { flakyTestRetriesCount: this._config.flakyTestRetriesCount, isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, isKnownTestsEnabled, - isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled + isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled, + testManagementAttemptToFixRetries } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 39d9fd1e11b..afae8f08486 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -12,6 +12,7 @@ const { const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2 const DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD = 30 +const DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES = 20 function getLibraryConfiguration ({ url, @@ -113,7 +114,9 @@ function getLibraryConfiguration ({ isFlakyTestRetriesEnabled, isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, isKnownTestsEnabled, - isTestManagementEnabled: (testManagementConfig?.enabled ?? false) + isTestManagementEnabled: (testManagementConfig?.enabled ?? false), + testManagementAttemptToFixRetries: + testManagementConfig?.attempt_to_fix_retries ?? DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) From ccbb60fe0d09bef7ddec37b623636137f8b18694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Mon, 17 Mar 2025 11:31:07 +0100 Subject: [PATCH 02/16] Add tag for attempt_to_fix --- packages/dd-trace/src/plugins/util/test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 92d62424d48..e4ff9959bc5 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -120,6 +120,7 @@ const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id' const DI_DEBUG_ERROR_FILE_SUFFIX = 'file' const DI_DEBUG_ERROR_LINE_SUFFIX = 'line' +const TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX = 'test.test_management.is_attempt_to_fix' const TEST_MANAGEMENT_IS_DISABLED = 'test.test_management.is_test_disabled' const TEST_MANAGEMENT_IS_QUARANTINED = 'test.test_management.is_quarantined' const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled' @@ -212,6 +213,7 @@ module.exports = { DI_DEBUG_ERROR_LINE_SUFFIX, getFormattedError, DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_DISABLED, TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED From 0d6f7c478ea594a3d9c3487db6710cce70fef4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Tue, 18 Mar 2025 10:11:37 +0100 Subject: [PATCH 03/16] Add configs and tags --- .../exporters/ci-visibility-exporter.js | 3 ++- .../requests/get-library-configuration.js | 3 +-- packages/dd-trace/src/plugins/util/test.js | 21 ++++++++++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index b615f862d2b..dbbcd2f0f0b 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -231,7 +231,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, isKnownTestsEnabled, isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled, - testManagementAttemptToFixRetries + testManagementAttemptToFixRetries: + testManagementAttemptToFixRetries ?? this._config.testManagementAttemptToFixRetries } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index afae8f08486..ebd00ea7574 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -12,7 +12,6 @@ const { const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2 const DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD = 30 -const DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES = 20 function getLibraryConfiguration ({ url, @@ -116,7 +115,7 @@ function getLibraryConfiguration ({ isKnownTestsEnabled, isTestManagementEnabled: (testManagementConfig?.enabled ?? false), testManagementAttemptToFixRetries: - testManagementConfig?.attempt_to_fix_retries ?? DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES + testManagementConfig?.attempt_to_fix_retries } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index e4ff9959bc5..c6e95d5a45b 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -58,6 +58,7 @@ const TEST_IS_RETRY = 'test.is_retry' const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' const TEST_RETRY_REASON = 'test.retry_reason' +const TEST_HAS_FAILED_ALL_RETRIES = 'test.has_failed_all_retries' const CI_APP_ORIGIN = 'ciapp-test' @@ -120,10 +121,16 @@ const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id' const DI_DEBUG_ERROR_FILE_SUFFIX = 'file' const DI_DEBUG_ERROR_LINE_SUFFIX = 'line' +// Test Management tags const TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX = 'test.test_management.is_attempt_to_fix' const TEST_MANAGEMENT_IS_DISABLED = 'test.test_management.is_test_disabled' const TEST_MANAGEMENT_IS_QUARANTINED = 'test.test_management.is_quarantined' const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled' +const TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED = 'test.test_management.attempt_to_fix_passed' + +// Test Management utils strings +const ATTEMPT_TO_FIX_STRING = "Retried by Datadog's Test Management" +const ATTEMPT_TEST_NAME_REGEX = new RegExp(ATTEMPT_TO_FIX_STRING + ' \\(#\\d+\\): ', 'g') module.exports = { TEST_CODE_OWNERS, @@ -156,6 +163,7 @@ module.exports = { TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, TEST_RETRY_REASON, + TEST_HAS_FAILED_ALL_RETRIES, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, @@ -193,7 +201,9 @@ module.exports = { EFD_STRING, EFD_TEST_NAME_REGEX, removeEfdStringFromTestName, + removeAttemptToFixStringFromTestName, addEfdStringToTestName, + addAttemptToFixStringToTestName, getIsFaultyEarlyFlakeDetection, TEST_BROWSER_DRIVER, TEST_BROWSER_DRIVER_VERSION, @@ -216,7 +226,8 @@ module.exports = { TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_DISABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_ENABLED + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 @@ -624,10 +635,18 @@ function addEfdStringToTestName (testName, numAttempt) { return `${EFD_STRING} (#${numAttempt}): ${testName}` } +function addAttemptToFixStringToTestName (testName, numAttempt) { + return `${ATTEMPT_TO_FIX_STRING} (#${numAttempt}): ${testName}` +} + function removeEfdStringFromTestName (testName) { return testName.replace(EFD_TEST_NAME_REGEX, '') } +function removeAttemptToFixStringFromTestName (testName) { + return testName.replace(ATTEMPT_TEST_NAME_REGEX, '') +} + function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) { let newSuites = 0 for (const suite of projectSuites) { From 750486e58105d99413db89024f3fc85683155d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Tue, 18 Mar 2025 12:20:08 +0100 Subject: [PATCH 04/16] Add jest support --- .../test-management/test-attempt-to-fix-1.js | 9 + .../test-management/test-attempt-to-fix-2.js | 9 + integration-tests/jest/jest.spec.js | 170 +++++++++++++++- packages/datadog-instrumentations/src/jest.js | 184 +++++++++++++----- packages/datadog-plugin-jest/src/index.js | 27 ++- 5 files changed, 346 insertions(+), 53 deletions(-) create mode 100644 integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js create mode 100644 integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js new file mode 100644 index 00000000000..113b2fd87ca --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +describe('attempt to fix tests', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix') // to check if this is being run + expect(1 + 2).to.equal(4) + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js new file mode 100644 index 00000000000..053d1d62eb0 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +describe('attempt to fix tests 2', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix 2') // to check if this is being run + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index 06708e793eb..eefeb91f178 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -46,7 +46,9 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2942,6 +2944,172 @@ describe('jest CommonJS', () => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptToFix, isParallel) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + if (isParallel) { + // Parallel mode in jest requires more than a single test suite + // Here we check that the second test suite is actually running, + // so we can be sure that parallel mode is on + const parallelTestName = 'ci-visibility/test-management/test-attempt-to-fix-2.js.' + + 'attempt to fix tests 2 can attempt to fix a test' + assert.includeMembers(resourceNames, [parallelTestName]) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (isAttemptToFix && i !== 0) { + // assert.equal(skippedTest.meta[TEST_STATUS], 'pass') + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + + if (isAttemptToFix && i === retriedTests.length - 1) { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + }) + + const runAttemptToFixTest = (done, isAttemptToFix, isQuarantined, extraEnvVars = {}, isParallel = false) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions(isAttemptToFix, isParallel) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-management/test-attempt-to-fix-1', + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + // jest uses stderr to output logs + childProcess.stderr.on('data', (chunk) => { + stdout += chunk.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (isAttemptToFix && isQuarantined) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, true, false) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runAttemptToFixTest(done, false, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, true, true) + }) + + it('can attempt to fix in parallel mode', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runAttemptToFixTest( + done, + true, + false, + { + // we need to run more than 1 suite for parallel mode to kick in + TESTS_TO_RUN: 'test-management/test-attempt-to-fix', + RUN_IN_PARALLEL: true + }, + true + ) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 1f16f98fe73..b7ca01024dd 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -13,7 +13,9 @@ const { addEfdStringToTestName, removeEfdStringFromTestName, getIsFaultyEarlyFlakeDetection, - JEST_WORKER_LOGS_PAYLOAD_CODE + JEST_WORKER_LOGS_PAYLOAD_CODE, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../dd-trace/src/plugins/util/test') const { getFormattedJestTestParameters, @@ -73,6 +75,7 @@ let hasFilteredSkippableSuites = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false let testManagementTests = {} +let testManagementAttemptToFixRetries = 0 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -80,6 +83,7 @@ const asyncResources = new WeakMap() const originalTestFns = new WeakMap() const retriedTestsToNumAttempts = new Map() const newTestsTestStatuses = new Map() +const attemptToFixRetriedTestsStatuses = new Map() const BREAKPOINT_HIT_GRACE_PERIOD_MS = 200 @@ -110,7 +114,7 @@ function getTestEnvironmentOptions (config) { return {} } -function getEfdStats (testStatuses) { +function getTestStats (testStatuses) { return testStatuses.reduce((acc, testStatus) => { acc[testStatus]++ return acc @@ -144,6 +148,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled this.isKnownTestsEnabled = this.testEnvironmentOptions._ddIsKnownTestsEnabled this.isTestManagementTestsEnabled = this.testEnvironmentOptions._ddIsTestManagementTestsEnabled + this.testManagementAttemptToFixRetries = this.testEnvironmentOptions._ddTestManagementAttemptToFixRetries if (this.isKnownTestsEnabled) { try { @@ -213,9 +218,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.testManagementTestsForThisSuite) { return this.testManagementTestsForThisSuite } - // TODO - ADD ATTEMPT_TO_FIX tests if (!testManagementTests) { return { + attempt_to_fix: [], disabled: [], quarantined: [] } @@ -228,12 +233,15 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } const result = { + attempt_to_fix: [], disabled: [], quarantined: [] } Object.entries(testManagementTestsForSuite).forEach(([testName, { properties }]) => { - if (properties?.disabled) { + if (properties?.attempt_to_fix) { + result.attempt_to_fix.push(testName) + } else if (properties?.disabled) { result.disabled.push(testName) } else if (properties?.quarantined) { result.quarantined.push(testName) @@ -243,6 +251,25 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { return result } + // Generic function to handle test retries + retryTest (testName, retryCount, addRetryStringToTestName, retryType, event) { + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of ${retryType} with the whole test suite + if (this.getHasSnapshotTests()) { + log.warn(`${retryType} is disabled for suites with snapshots`) + return + } + + for (let retryIndex = 0; retryIndex < retryCount; retryIndex++) { + if (this.global.test) { + this.global.test(addRetryStringToTestName(testName, retryIndex), event.fn, event.timeout) + } else { + log.error(`${retryType} could not retry test because global.test is undefined`) + } + } + } + // Add the `add_test` event we don't have the test object yet, so getTestNameFromAddTestEvent (event, state) { const describeSuffix = getJestTestName(state.currentDescribeBlock) @@ -273,15 +300,30 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (event.name === 'test_start') { let isNewTest = false let numEfdRetry = null + let numOfAttemptsToFixRetries = null const testParameters = getTestParametersString(this.nameToParams, event.test.name) // Async resource for this test is created here // It is used later on by the test_done handler const asyncResource = new AsyncResource('bound-anonymous-fn') asyncResources.set(event.test, asyncResource) const testName = getJestTestName(event.test) + let originalTestName = testName - if (this.isKnownTestsEnabled) { - const originalTestName = removeEfdStringFromTestName(testName) + let isAttemptToFix = false + if (this.isTestManagementTestsEnabled) { + originalTestName = removeAttemptToFixStringFromTestName(testName) + isAttemptToFix = this.testManagementTestsForThisSuite?.attempt_to_fix?.includes(originalTestName) + const isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(originalTestName) + if (isAttemptToFix) { + numOfAttemptsToFixRetries = retriedTestsToNumAttempts.get(originalTestName) + retriedTestsToNumAttempts.set(originalTestName, numOfAttemptsToFixRetries + 1) + } else if (isDisabled) { + event.test.mode = 'skip' + } + } + + if (this.isKnownTestsEnabled && !isAttemptToFix) { + originalTestName = removeEfdStringFromTestName(testName) isNewTest = retriedTestsToNumAttempts.has(originalTestName) if (isNewTest) { numEfdRetry = retriedTestsToNumAttempts.get(originalTestName) @@ -289,16 +331,10 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } - if (this.isTestManagementTestsEnabled) { - const isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(testName) - if (isDisabled) { - event.test.mode = 'skip' - } - } const isJestRetry = event.test?.invocations > 1 asyncResource.runInAsyncScope(() => { testStartCh.publish({ - name: removeEfdStringFromTestName(testName), + name: originalTestName, suite: this.testSuite, testSourceFile: this.testSourceFile, displayName: this.displayName, @@ -306,6 +342,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { frameworkVersion: jestVersion, isNew: isNewTest, isEfdRetry: numEfdRetry > 0, + isAttemptToFixRetry: numOfAttemptsToFixRetries > 0, isJestRetry }) originalTestFns.set(event.test, event.test.fn) @@ -313,27 +350,36 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { }) } if (event.name === 'add_test') { - if (this.isKnownTestsEnabled) { - const testName = this.getTestNameFromAddTestEvent(event, state) - const isNew = !this.knownTestsForThisSuite?.includes(testName) - const isSkipped = event.mode === 'todo' || event.mode === 'skip' - if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { - retriedTestsToNumAttempts.set(testName, 0) + const testName = this.getTestNameFromAddTestEvent(event, state) + let originalTestName = testName + const isSkipped = event.mode === 'todo' || event.mode === 'skip' + let isAttemptToFix = false + if (this.isTestManagementTestsEnabled) { + originalTestName = removeAttemptToFixStringFromTestName(testName) + isAttemptToFix = this.testManagementTestsForThisSuite?.attempt_to_fix?.includes(originalTestName) + if (!isSkipped && !retriedTestsToNumAttempts.has(originalTestName) && isAttemptToFix) { + retriedTestsToNumAttempts.set(originalTestName, 0) + this.retryTest( + event.testName, + testManagementAttemptToFixRetries, + addAttemptToFixStringToTestName, + 'Test Management (Attempt to Fix)', + event + ) + } + } + if (this.isKnownTestsEnabled && !isAttemptToFix) { + const isNew = !this.knownTestsForThisSuite?.includes(originalTestName) + if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(originalTestName)) { + retriedTestsToNumAttempts.set(originalTestName, 0) if (this.isEarlyFlakeDetectionEnabled) { - // Retrying snapshots has proven to be problematic, so we'll skip them for now - // We'll still detect new tests, but we won't retry them. - // TODO: do not bail out of EFD with the whole test suite - if (this.getHasSnapshotTests()) { - log.warn('Early flake detection is disabled for suites with snapshots') - return - } - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - if (this.global.test) { - this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) - } else { - log.error('Early flake detection could not retry test because global.test is undefined') - } - } + this.retryTest( + event.testName, + earlyFlakeDetectionNumRetries, + addEfdStringToTestName, + 'Early flake detection', + event + ) } } } @@ -346,8 +392,41 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { // restore in case it is retried event.test.fn = originalTestFns.get(event.test) + let isAttemptToFix = false + let isQuarantined = false + let attemptToFixPassed = false + let failedAllTests = false + if (this.isTestManagementTestsEnabled) { + const testName = getJestTestName(event.test) + const originalTestName = removeAttemptToFixStringFromTestName(testName) + isAttemptToFix = this.testManagementTestsForThisSuite?.attempt_to_fix?.includes(originalTestName) + if (isAttemptToFix) { + if (attemptToFixRetriedTestsStatuses.has(originalTestName)) { + attemptToFixRetriedTestsStatuses.get(originalTestName).push(status) + } else { + attemptToFixRetriedTestsStatuses.set(originalTestName, [status]) + } + // Check if this is the last attempt to fix. + // If it is, we'll set the failedAllTests flag to true if all the tests failed + // If all tests passed, we'll set the attemptToFixPassed flag to true + // The +1 is because the first attempt is not counted as a retry + const numAttempts = testManagementAttemptToFixRetries + 1 + if (attemptToFixRetriedTestsStatuses.get(originalTestName).length === numAttempts) { + const { pass, fail } = getTestStats(attemptToFixRetriedTestsStatuses.get(originalTestName)) + + if (pass === numAttempts) { + attemptToFixPassed = true + } else if (fail === numAttempts) { + failedAllTests = true + } + } + } else { + isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(testName) + } + } + // We'll store the test statuses of the retries - if (this.isKnownTestsEnabled) { + if (this.isKnownTestsEnabled && !isAttemptToFix) { const testName = getJestTestName(event.test) const originalTestName = removeEfdStringFromTestName(testName) const isNewTest = retriedTestsToNumAttempts.has(originalTestName) @@ -359,12 +438,6 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } } - let isQuarantined = false - - if (this.isTestManagementTestsEnabled) { - const testName = getJestTestName(event.test) - isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(testName) - } const promises = {} const numRetries = this.global[RETRY_TIMES] @@ -399,7 +472,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { testFinishCh.publish({ status, testStartLine: getTestLineStart(event.test.asyncError, this.testSuite), - isQuarantined + isQuarantined, + attemptToFixPassed, + failedAllTests }) }) @@ -552,6 +627,7 @@ function cliWrapper (cli, jestVersion) { earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (err) { log.error('Jest library configuration error', err) @@ -710,7 +786,7 @@ function cliWrapper (cli, jestVersion) { if (isEarlyFlakeDetectionEnabled) { let numFailedTestsToIgnore = 0 for (const testStatuses of newTestsTestStatuses.values()) { - const { pass, fail } = getEfdStats(testStatuses) + const { pass, fail } = getTestStats(testStatuses) if (pass > 0) { // as long as one passes, we'll consider the test passed numFailedTestsToIgnore += fail } @@ -724,30 +800,37 @@ function cliWrapper (cli, jestVersion) { if (isTestManagementTestsEnabled) { const failedTests = result .results - .testResults.flatMap(({ testResults, testFilePath: testSuiteAbsolutePath }) => ( - testResults.map(({ fullName: testName, status }) => ({ testName, testSuiteAbsolutePath, status })) + .testResults.flatMap(({ testResults, testFilePath: testSuiteAbsolutePath }, firstIndex) => ( + testResults.map(({ fullName: testName, status }, secondIndex) => ( + { testName, testSuiteAbsolutePath, status, firstIndex, secondIndex } + )) )) .filter(({ status }) => status === 'failed') let numFailedQuarantinedTests = 0 + let numFailedRetriedQuarantinedOrDisabledTests = 0 - for (const { testName, testSuiteAbsolutePath } of failedTests) { + for (const { testName, testSuiteAbsolutePath, firstIndex, secondIndex } of failedTests) { const testSuite = getTestSuitePath(testSuiteAbsolutePath, result.globalConfig.rootDir) - const isQuarantined = testManagementTests + const originalName = removeAttemptToFixStringFromTestName(testName) + const testManagementTest = testManagementTests ?.jest ?.suites ?.[testSuite] ?.tests - ?.[testName] + ?.[originalName] ?.properties - ?.quarantined - if (isQuarantined) { + if (testManagementTest?.attempt_to_fix && (testManagementTest?.quarantined || testManagementTest?.disabled)) { + numFailedRetriedQuarantinedOrDisabledTests++ + result.results.testResults[firstIndex].testResults[secondIndex].status = 'pass' + } else if (testManagementTest?.quarantined) { numFailedQuarantinedTests++ } } // If every test that failed was quarantined, we'll consider the suite passed - if (numFailedQuarantinedTests !== 0 && result.results.numFailedTests === numFailedQuarantinedTests) { + if ((numFailedRetriedQuarantinedOrDisabledTests !== 0 || numFailedQuarantinedTests !== 0) && + result.results.numFailedTests === numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests) { result.results.success = true } } @@ -947,6 +1030,7 @@ addHook({ _ddIsKnownTestsEnabled, _ddIsTestManagementTestsEnabled, _ddTestManagementTests, + _ddTestManagementAttemptToFixRetries, ...restOfTestEnvironmentOptions } = testEnvironmentOptions diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 1e231443772..4c6dc2e240c 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -27,7 +27,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -177,6 +180,7 @@ class JestPlugin extends CiPlugin { config._ddRepositoryRoot = this.repositoryRoot config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddIsTestManagementTestsEnabled = this.libraryConfig?.isTestManagementEnabled ?? false + config._ddTestManagementAttemptToFixRetries = this.libraryConfig?.testManagementAttemptToFixRetries ?? 0 config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false @@ -336,7 +340,13 @@ class JestPlugin extends CiPlugin { this.activeTestSpan = span }) - this.addSub('ci:jest:test:finish', ({ status, testStartLine, isQuarantined }) => { + this.addSub('ci:jest:test:finish', ({ + status, + testStartLine, + isQuarantined, + attemptToFixPassed, + failedAllTests + }) => { const span = storage('legacy').getStore().span span.setTag(TEST_STATUS, status) if (testStartLine) { @@ -345,6 +355,12 @@ class JestPlugin extends CiPlugin { if (isQuarantined) { span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') } + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + if (failedAllTests) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( @@ -407,6 +423,7 @@ class JestPlugin extends CiPlugin { testSourceFile, isNew, isEfdRetry, + isAttemptToFixRetry, isJestRetry } = test @@ -425,6 +442,12 @@ class JestPlugin extends CiPlugin { extraTags[JEST_DISPLAY_NAME] = displayName } + if (isAttemptToFixRetry) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } + if (isNew) { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { From 2740adaa9cd2247dfd44cc0c20ddf4c3d227d7e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Tue, 18 Mar 2025 14:52:40 +0100 Subject: [PATCH 05/16] Fix paralel jest --- integration-tests/jest/jest.spec.js | 9 ++++----- packages/datadog-instrumentations/src/jest.js | 14 +++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index eefeb91f178..eddd69b9128 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -3000,7 +3000,6 @@ describe('jest CommonJS', () => { for (let i = 0; i < retriedTests.length; i++) { const test = retriedTests[i] if (isAttemptToFix && i !== 0) { - // assert.equal(skippedTest.meta[TEST_STATUS], 'pass') assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') @@ -3060,19 +3059,19 @@ describe('jest CommonJS', () => { }) it('does not attempt to fix tests if test management is not enabled', (done) => { - receiver.setSettings({ test_management: { enabled: false } }) + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) runAttemptToFixTest(done, false, false) }) it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { - receiver.setSettings({ test_management: { enabled: true } }) + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) }) it('does not fail retry if a test is quarantined', (done) => { - receiver.setSettings({ test_management: { enabled: true } }) + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) receiver.setTestManagementTests({ jest: { suites: { @@ -3094,7 +3093,7 @@ describe('jest CommonJS', () => { }) it('can attempt to fix in parallel mode', (done) => { - receiver.setSettings({ test_management: { enabled: true } }) + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) runAttemptToFixTest( done, diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index b7ca01024dd..4563f246092 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -148,7 +148,6 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled this.isKnownTestsEnabled = this.testEnvironmentOptions._ddIsKnownTestsEnabled this.isTestManagementTestsEnabled = this.testEnvironmentOptions._ddIsTestManagementTestsEnabled - this.testManagementAttemptToFixRetries = this.testEnvironmentOptions._ddTestManagementAttemptToFixRetries if (this.isKnownTestsEnabled) { try { @@ -174,6 +173,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.isTestManagementTestsEnabled) { try { const hasTestManagementTests = !!testManagementTests.jest + testManagementAttemptToFixRetries = this.testEnvironmentOptions._ddTestManagementAttemptToFixRetries this.testManagementTestsForThisSuite = hasTestManagementTests ? this.getTestManagementTestsForSuite(testManagementTests.jest.suites?.[this.testSuite]?.tests) : this.getTestManagementTestsForSuite(this.testEnvironmentOptions._ddTestManagementTests) @@ -409,14 +409,14 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { // Check if this is the last attempt to fix. // If it is, we'll set the failedAllTests flag to true if all the tests failed // If all tests passed, we'll set the attemptToFixPassed flag to true - // The +1 is because the first attempt is not counted as a retry - const numAttempts = testManagementAttemptToFixRetries + 1 - if (attemptToFixRetriedTestsStatuses.get(originalTestName).length === numAttempts) { - const { pass, fail } = getTestStats(attemptToFixRetriedTestsStatuses.get(originalTestName)) + if (attemptToFixRetriedTestsStatuses.get(originalTestName).length === + testManagementAttemptToFixRetries + 1) { + // The slice is because the first attempt is not counted as a retry + const { pass, fail } = getTestStats(attemptToFixRetriedTestsStatuses.get(originalTestName).slice(1)) - if (pass === numAttempts) { + if (pass === testManagementAttemptToFixRetries) { attemptToFixPassed = true - } else if (fail === numAttempts) { + } else if (fail === testManagementAttemptToFixRetries) { failedAllTests = true } } From cc8ac7a958a571587c87f589bc1860b36b89d681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Tue, 18 Mar 2025 15:56:46 +0100 Subject: [PATCH 06/16] Add cucumber support --- .../features-test-management/support/steps.js | 7 + integration-tests/cucumber/cucumber.spec.js | 137 +++++++++++++++++- .../datadog-instrumentations/src/cucumber.js | 103 +++++++++---- packages/datadog-plugin-cucumber/src/index.js | 21 ++- 4 files changed, 240 insertions(+), 28 deletions(-) diff --git a/integration-tests/ci-visibility/features-test-management/support/steps.js b/integration-tests/ci-visibility/features-test-management/support/steps.js index e01c21e968a..2fb35e92140 100644 --- a/integration-tests/ci-visibility/features-test-management/support/steps.js +++ b/integration-tests/ci-visibility/features-test-management/support/steps.js @@ -21,3 +21,10 @@ When('the greeter says disabled', function () { // expected to fail if not disabled this.whatIHeard = 'disabld' }) + +When('the greeter says attempt to fix', function () { + // eslint-disable-next-line no-console + console.log('I am running') // just to assert whether this is running + // expected to fail + this.whatIHeard = 'attempt to fx' +}) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 2359577b9ea..99bb6f442db 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -49,7 +49,9 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -2031,6 +2033,139 @@ versions.forEach(version => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptToFix) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'Say attempt to fix' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + + const testResource = 'ci-visibility/features-test-management/attempt-to-fix.feature.' + + 'Say attempt to fix' + assert.equal(test.resource, testResource) + + if (isAttemptToFix && i !== 0) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + + if (isAttemptToFix && i === retriedTests.length - 1) { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + }) + + const runTest = (done, isAttemptToFix, isQuarantined, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isAttemptToFix) + let stdout = '' + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running') + if (isQuarantined) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, true, false) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runTest(done, false, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runTest(done, true, true) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index ce82c268e3f..9a977af8f54 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -73,6 +73,7 @@ let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} let numTestRetries = 0 let knownTests = [] @@ -121,10 +122,10 @@ function isNewTest (testSuite, testName) { } function getTestProperties (testSuite, testName) { - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.cucumber?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function getTestStatusFromRetries (testStatuses) { @@ -303,22 +304,46 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false + let isAttemptToFix = false + let isAttemptToFixRetry = false + let hasFailedAllRetries = false + let hasPassedAllRetries = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled && status !== 'skip') { - const numRetries = numRetriesByPickleId.get(this.pickle.id) - - isNew = numRetries !== undefined - isEfdRetry = numRetries > 0 - } if (isTestManagementTestsEnabled) { const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) const testProperties = getTestProperties(testSuitePath, this.pickle.name) - isDisabled = testProperties.disabled - if (!isDisabled) { - isQuarantined = testProperties.quarantined + const numRetries = numRetriesByPickleId.get(this.pickle.id) + isAttemptToFix = testProperties.attemptToFix + isAttemptToFixRetry = isAttemptToFix && numRetries > 0 + if (!isAttemptToFix) { + isDisabled = testProperties.disabled + if (!isDisabled) { + isQuarantined = testProperties.quarantined + } + } + + if (isAttemptToFixRetry) { + const statuses = lastStatusByPickleId.get(this.pickle.id) + if (statuses.length === testManagementAttemptToFixRetries + 1) { + // The first status is the original test status + const statusesExceptFirst = statuses.slice(1) + const { pass, fail } = statusesExceptFirst.reduce((acc, status) => { + acc[status]++ + return acc + }, { pass: 0, fail: 0 }) + hasFailedAllRetries = fail === testManagementAttemptToFixRetries + hasPassedAllRetries = pass === testManagementAttemptToFixRetries + } } } + + if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) { + const numRetries = numRetriesByPickleId.get(this.pickle.id) + + isNew = numRetries !== undefined + isEfdRetry = numRetries > 0 + } const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) const error = getErrorFromCucumberResult(result) @@ -334,6 +359,9 @@ function wrapRun (pl, isLatestVersion) { isNew, isEfdRetry, isFlakyRetry: numAttempt > 0, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) @@ -426,6 +454,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled isTestManagementTestsEnabled = configurationResponse.libraryConfig?.isTestManagementEnabled + testManagementAttemptToFixRetries = configurationResponse.libraryConfig?.testManagementAttemptToFixRetries if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) @@ -576,29 +605,42 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let isNew = false + let isAttemptToFix = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled) { + if (isTestManagementTestsEnabled) { + const testProperties = getTestProperties(testSuitePath, pickle.name) + isAttemptToFix = testProperties.attemptToFix + if (!isAttemptToFix) { + isDisabled = testProperties.disabled + if (isDisabled) { + this.options.dryRun = true + } else { + isQuarantined = testProperties.quarantined + } + } + } + + if (isKnownTestsEnabled && !isAttemptToFix) { isNew = isNewTest(testSuitePath, pickle.name) if (isNew) { numRetriesByPickleId.set(pickle.id, 0) } } - if (isTestManagementTestsEnabled) { - const testProperties = getTestProperties(testSuitePath, pickle.name) - isDisabled = testProperties.disabled - if (isDisabled) { - this.options.dryRun = true - } else { - isQuarantined = testProperties.quarantined - } - } // TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId` let runTestCaseResult = await runTestCaseFunction.apply(this, arguments) const testStatuses = lastStatusByPickleId.get(pickle.id) const lastTestStatus = testStatuses[testStatuses.length - 1] + + if (isAttemptToFix && lastTestStatus !== 'skip') { + for (let retryIndex = 0; retryIndex < testManagementAttemptToFixRetries; retryIndex++) { + numRetriesByPickleId.set(pickle.id, retryIndex + 1) + runTestCaseResult = await runTestCaseFunction.apply(this, arguments) + } + } + // If it's a new test and it hasn't been skipped, we run it again if (isEarlyFlakeDetectionEnabled && lastTestStatus !== 'skip' && isNew) { for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { @@ -609,6 +651,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa let testStatus = lastTestStatus let shouldBePassedByEFD = false let shouldBePassedByQuarantine = false + let shouldBePassedByAttemptToFix = false if (isNew && isEarlyFlakeDetectionEnabled) { /** * If Early Flake Detection (EFD) is enabled the logic is as follows: @@ -625,9 +668,17 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } } - if (isTestManagementTestsEnabled && isQuarantined) { - this.success = true - shouldBePassedByQuarantine = true + if (isTestManagementTestsEnabled) { + if (isAttemptToFix) { + const testProperties = getTestProperties(testSuitePath, pickle.name) + if (testProperties.disabled || testProperties.quarantined) { + this.success = true + shouldBePassedByAttemptToFix = true + } + } else if (isQuarantined) { + this.success = true + shouldBePassedByQuarantine = true + } } if (!pickleResultByFile[testFileAbsolutePath]) { @@ -661,8 +712,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa return shouldBePassedByEFD } - if (isNewerCucumberVersion && isTestManagementTestsEnabled && isQuarantined) { - return shouldBePassedByQuarantine + if (isNewerCucumberVersion && isTestManagementTestsEnabled && (isQuarantined || isAttemptToFix)) { + return shouldBePassedByQuarantine || shouldBePassedByAttemptToFix } return runTestCaseResult diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index f7ff7e4c2e9..0ea1239fd15 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -30,7 +30,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -328,6 +331,9 @@ class CucumberPlugin extends CiPlugin { isNew, isEfdRetry, isFlakyRetry, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) => { @@ -358,6 +364,19 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + + if (isAttemptToFixRetry) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + if (hasPassedAllRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + if (isDisabled) { span.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true') } From becd181a458a6f5df4a74365ca3b8d5158b8b381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Tue, 18 Mar 2025 15:57:09 +0100 Subject: [PATCH 07/16] Add missing test --- .../features-test-management/attempt-to-fix.feature | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature diff --git a/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature new file mode 100644 index 00000000000..98ac42a8213 --- /dev/null +++ b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature @@ -0,0 +1,4 @@ +Feature: Attempt to fix + Scenario: Say attempt to fix + When the greeter says attempt to fix + Then I should have heard "attempt to fix" From ee8d4cc1d077a5c014569bf4f6dccde72a459da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 19 Mar 2025 10:55:38 +0100 Subject: [PATCH 08/16] Fix cucumber conditional --- packages/datadog-instrumentations/src/cucumber.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index 9a977af8f54..1b3a3f53306 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -338,11 +338,13 @@ function wrapRun (pl, isLatestVersion) { } } - if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) { + if (isKnownTestsEnabled && status !== 'skip') { const numRetries = numRetriesByPickleId.get(this.pickle.id) isNew = numRetries !== undefined - isEfdRetry = numRetries > 0 + if (!isAttemptToFix) { + isEfdRetry = numRetries > 0 + } } const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) From a36f273751bd2ca81d2179e14c2738b9e92c8db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 19 Mar 2025 12:34:53 +0100 Subject: [PATCH 09/16] Add commit message to the request --- .../datadog-plugin-cypress/src/cypress-plugin.js | 9 ++++++--- .../test-management/get-test-management-tests.js | 6 ++++-- packages/dd-trace/src/plugins/ci_plugin.js | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 8767b6b9016..de9e56f5c7a 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -64,7 +64,8 @@ const { GIT_COMMIT_SHA, GIT_BRANCH, CI_PROVIDER_NAME, - CI_WORKSPACE_PATH + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE } = require('../../dd-trace/src/plugins/util/tags') const { OS_VERSION, @@ -201,7 +202,8 @@ class CypressPlugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot @@ -217,7 +219,8 @@ class CypressPlugin { runtimeName, runtimeVersion, branch, - testLevel: 'test' + testLevel: 'test', + commitMessage } this.finishedTestsByFile = {} diff --git a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js index 5aca25e9e19..f5e897f0782 100644 --- a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js +++ b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js @@ -6,7 +6,8 @@ function getTestManagementTests ({ isEvpProxy, evpProxyPrefix, isGzipCompatible, - repositoryUrl + repositoryUrl, + commitMessage }, done) { const options = { path: '/api/v2/test/libraries/test-management/tests', @@ -39,7 +40,8 @@ function getTestManagementTests ({ id: id().toString(10), type: 'ci_app_libraries_tests_request', attributes: { - repository_url: repositoryUrl + repository_url: repositoryUrl, + commit_message: commitMessage } } }) diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index e1940265706..63c716acb32 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -41,7 +41,14 @@ const { TELEMETRY_EVENT_CREATED, TELEMETRY_ITR_SKIPPED } = require('../ci-visibility/telemetry') -const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH, CI_WORKSPACE_PATH } = require('./util/tags') +const { + CI_PROVIDER_NAME, + GIT_REPOSITORY_URL, + GIT_COMMIT_SHA, + GIT_BRANCH, + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE +} = require('./util/tags') const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') const getDiClient = require('../ci-visibility/dynamic-instrumentation') @@ -251,7 +258,8 @@ module.exports = class CiPlugin extends Plugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot || process.cwd() @@ -269,7 +277,8 @@ module.exports = class CiPlugin extends Plugin { runtimeName, runtimeVersion, branch, - testLevel: 'suite' + testLevel: 'suite', + commitMessage } } From d394f546ac55f3f31d90f0ba39e1ca7421ce4c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 19 Mar 2025 14:13:52 +0100 Subject: [PATCH 10/16] Undo cucumber condiotional change --- packages/datadog-instrumentations/src/cucumber.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index 1b3a3f53306..9a977af8f54 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -338,13 +338,11 @@ function wrapRun (pl, isLatestVersion) { } } - if (isKnownTestsEnabled && status !== 'skip') { + if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) { const numRetries = numRetriesByPickleId.get(this.pickle.id) isNew = numRetries !== undefined - if (!isAttemptToFix) { - isEfdRetry = numRetries > 0 - } + isEfdRetry = numRetries > 0 } const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) From 36290ba36e9b8a3b199ff56b3c98c7ee7f2bee8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 19 Mar 2025 17:33:01 +0100 Subject: [PATCH 11/16] Add support for Mocha (no parallel mode) --- integration-tests/mocha/mocha.spec.js | 145 +++++++++++++++++- .../src/mocha/main.js | 39 +++-- .../src/mocha/utils.js | 112 ++++++++++---- .../src/mocha/worker.js | 1 + packages/datadog-plugin-mocha/src/index.js | 30 +++- 5 files changed, 283 insertions(+), 44 deletions(-) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 6c96d2fb136..0d15bbc0b5c 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -48,7 +48,9 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2566,6 +2568,147 @@ describe('mocha CommonJS', function () { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptToFix) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (isAttemptToFix && i !== 0) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + + if (isAttemptToFix && i === retriedTests.length - 1) { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + }) + + const runAttemptToFixTest = (done, isAttemptToFix, isQuarantined, extraEnvVars = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions(isAttemptToFix) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-management/test-attempt-to-fix-1.js' + ]), + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (isAttemptToFix && isQuarantined) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, true, false) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, true, true) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 6589e3ccacb..0c136586c18 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -30,7 +30,9 @@ const { newTests, testsQuarantined, getTestFullName, - getRunTestsWrapper + getRunTestsWrapper, + testsAttemptToFix, + testsStatuses } = require('./utils') require('./common') @@ -138,16 +140,26 @@ function getOnEndHandler (isParallel) { } } + // We substract the errors of attempt to fix tests (quarantined or disabled) from the total number of failures // We subtract the errors from quarantined tests from the total number of failures if (config.isTestManagementTestsEnabled) { let numFailedQuarantinedTests = 0 + let numFailedRetriedQuarantinedOrDisabledTests = 0 + for (const test of testsAttemptToFix) { + const testName = getTestFullName(test, true, false) + const testProperties = getTestProperties(test, config.testManagementTests) + if (isTestFailed(test) && (testProperties.isQuarantined || testProperties.isDisabled)) { + const failedTests = testsStatuses.get(testName).filter(status => status === 'fail') + numFailedRetriedQuarantinedOrDisabledTests += failedTests.length + } + } for (const test of testsQuarantined) { if (isTestFailed(test)) { numFailedQuarantinedTests++ } } - this.stats.failures -= numFailedQuarantinedTests - this.failures -= numFailedQuarantinedTests + this.stats.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests + this.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests } if (status === 'fail') { @@ -193,6 +205,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { if (err) { config.testManagementTests = {} config.isTestManagementTestsEnabled = false + config.testManagementAttemptToFixRetries = 0 } else { config.testManagementTests = receivedTestManagementTests } @@ -260,6 +273,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled config.isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + config.testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries // ITR and auto test retries are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled @@ -637,6 +651,7 @@ addHook({ if (config.isTestManagementTestsEnabled) { const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {} newWorkerArgs._ddIsTestManagementTestsEnabled = true + newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries newWorkerArgs._ddTestManagementTests = { mocha: { suites: { @@ -661,9 +676,19 @@ addHook({ .map(event => event.data) for (const test of tests) { + const testProperties = getTestProperties(test, config.testManagementTests) + if (config.isTestManagementTestsEnabled) { + // `testsAttemptToFix` is filled in the worker process, so we need to use the test results to fill it here too. + // `testsQuarantined` is filled in the worker process, so we need to use the test results to fill it here too. + if (testProperties.isAttemptToFix) { + testsAttemptToFix.add(test) + } else if (testProperties.isQuarantined) { + testsQuarantined.add(test) + } + } // `newTests` is filled in the worker process, so we need to use the test results to fill it here too. - if (config.isKnownTestsEnabled && isNewTest(test, config.knownTests)) { - const testFullName = getTestFullName(test) + if (config.isKnownTestsEnabled && isNewTest(test, config.knownTests) && !testProperties.isAttemptToFix) { + const testFullName = getTestFullName(test, false, true) const tests = newTests[testFullName] if (!tests) { @@ -672,10 +697,6 @@ addHook({ tests.push(test) } } - // `testsQuarantined` is filled in the worker process, so we need to use the test results to fill it here too. - if (config.isTestManagementTestsEnabled && getTestProperties(test, config.testManagementTests).isQuarantined) { - testsQuarantined.add(test) - } } return testFileResult }) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index ce33d1cf7c4..77df4645cdc 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -3,7 +3,9 @@ const { getTestSuitePath, removeEfdStringFromTestName, - addEfdStringToTestName + addEfdStringToTestName, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../../dd-trace/src/plugins/util/test') const { channel, AsyncResource } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') @@ -26,7 +28,9 @@ const testToStartLine = new WeakMap() const testFileToSuiteAr = new Map() const wrappedFunctions = new WeakSet() const newTests = {} +const testsAttemptToFix = new Set() const testsQuarantined = new Set() +const testsStatuses = new Map() function getAfterEachHooks (testOrHook) { const hooks = [] @@ -44,10 +48,10 @@ function getTestProperties (test, testManagementTests) { const testSuite = getTestSuitePath(test.file, process.cwd()) const testName = test.fullTitle() - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.mocha?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } function isNewTest (test, knownTests) { @@ -57,15 +61,24 @@ function isNewTest (test, knownTests) { return !testsForSuite.includes(testName) } -function retryTest (test, earlyFlakeDetectionNumRetries) { +function retryTest (test, numRetries, isAttemptToFix, isEfd) { const originalTestName = test.title const suite = test.parent - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() - clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + if (isAttemptToFix) { + clonedTest.title = addAttemptToFixStringToTestName(originalTestName, retryIndex + 1) + } else if (isEfd) { + clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + } suite.addTest(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + if (isAttemptToFix) { + clonedTest._ddIsAttemptToFix = true + clonedTest._ddIsLastRetry = retryIndex === numRetries - 1 + } else if (isEfd) { + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } } } @@ -101,8 +114,14 @@ function getIsLastRetry (test) { return test._currentRetry === test._retries } -function getTestFullName (test) { - return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` +function getTestFullName (test, isAttemptToFix, isEfdRetry) { + let testName = test.fullTitle() + if (isAttemptToFix) { + testName = removeAttemptToFixStringFromTestName(testName) + } else if (isEfdRetry) { + testName = removeEfdStringFromTestName(testName) + } + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${testName}` } function getTestStatus (test) { @@ -195,12 +214,19 @@ function getOnTestHandler (isMain) { title, _ddIsNew: isNew, _ddIsEfdRetry: isEfdRetry, + _ddIsAttemptToFix: isAttemptToFix, _ddIsDisabled: isDisabled, _ddIsQuarantined: isQuarantined } = test + let testName = test.fullTitle() + if (isAttemptToFix) { + testName = removeAttemptToFixStringFromTestName(testName) + } else if (isEfdRetry) { + testName = removeEfdStringFromTestName(testName) + } const testInfo = { - testName: test.fullTitle(), + testName, testSuiteAbsolutePath, title, testStartLine @@ -212,11 +238,12 @@ function getOnTestHandler (isMain) { testInfo.isNew = isNew testInfo.isEfdRetry = isEfdRetry + testInfo.isAttemptToFix = isAttemptToFix testInfo.isDisabled = isDisabled testInfo.isQuarantined = isQuarantined // We want to store the result of the new tests if (isNew) { - const testFullName = getTestFullName(test) + const testFullName = getTestFullName(test, false, true) if (newTests[testFullName]) { newTests[testFullName].push(test) } else { @@ -249,13 +276,32 @@ function getOnTestEndHandler () { }) } + let hasFailedAllRetries = false + let attemptToFixPassed = false + let testName = getTestFullName(test, false, false) + if (test._ddIsAttemptToFix) { + testName = getTestFullName(test, true, false) + } + + testsStatuses.set(testName, [...(testsStatuses.get(testName) || []), status]) + + if (test._ddIsAttemptToFix && test._ddIsLastRetry) { + if (testsStatuses.get(testName).every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (testsStatuses.get(testName).every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + // if there are afterEach to be run, we don't finish the test yet if (asyncResource && !getAfterEachHooks(test).length) { asyncResource.runInAsyncScope(() => { testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test), - isLastRetry: getIsLastRetry(test) + isLastRetry: getIsLastRetry(test), + hasFailedAllRetries, + attemptToFixPassed }) }) } @@ -374,29 +420,35 @@ function getOnPendingHandler () { } } -// Hook to add retries to tests if EFD is enabled +// Hook to add retries to tests if Test Management or EFD is enabled function getRunTestsWrapper (runTests, config) { return function (suite, fn) { - if (config.isKnownTestsEnabled) { - // by the time we reach `this.on('test')`, it is too late. We need to add retries here - suite.tests.forEach(test => { - if (!test.isPending() && isNewTest(test, config.knownTests)) { - test._ddIsNew = true - if (config.isEarlyFlakeDetectionEnabled) { - retryTest(test, config.earlyFlakeDetectionNumRetries) + if (config.isTestManagementTestsEnabled) { + suite.tests.forEach((test) => { + const { isAttemptToFix, isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) + if (isAttemptToFix) { + if (!test.isPending()) { + // This is needed to know afterwards which ones have been retried + testsAttemptToFix.add(test) + retryTest(test, config.testManagementAttemptToFixRetries, true, false) } + } else if (isDisabled) { + test._ddIsDisabled = true + } else if (isQuarantined) { + testsQuarantined.add(test) + test._ddIsQuarantined = true } }) } - if (config.isTestManagementTestsEnabled) { + if (config.isKnownTestsEnabled) { + // by the time we reach `this.on('test')`, it is too late. We need to add retries here suite.tests.forEach(test => { - const { isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) - if (isDisabled) { - test._ddIsDisabled = true - } else if (isQuarantined) { - testsQuarantined.add(test) - test._ddIsQuarantined = true + if (!test.isPending() && isNewTest(test, config.knownTests)) { + test._ddIsNew = true + if (config.isEarlyFlakeDetectionEnabled && !testsAttemptToFix.has(test)) { + retryTest(test, config.earlyFlakeDetectionNumRetries, false, true) + } } }) } @@ -427,5 +479,7 @@ module.exports = { testFileToSuiteAr, getRunTestsWrapper, newTests, - testsQuarantined + testsQuarantined, + testsAttemptToFix, + testsStatuses } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index d456150036f..37634df379b 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -36,6 +36,7 @@ addHook({ } if (this.options._ddIsTestManagementTestsEnabled) { config.isTestManagementTestsEnabled = true + config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries config.testManagementTests = this.options._ddTestManagementTests delete this.options._ddIsTestManagementTestsEnabled delete this.options._ddTestManagementTests diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 9df19700292..3841d5617f8 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -17,7 +17,6 @@ const { TEST_CODE_OWNERS, ITR_CORRELATION_ID, TEST_SOURCE_FILE, - removeEfdStringFromTestName, TEST_IS_NEW, TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, @@ -34,7 +33,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -199,7 +201,13 @@ class MochaPlugin extends CiPlugin { this.tracer._exporter.flush() }) - this.addSub('ci:mocha:test:finish', ({ status, hasBeenRetried, isLastRetry }) => { + this.addSub('ci:mocha:test:finish', ({ + status, + hasBeenRetried, + isLastRetry, + hasFailedAllRetries, + attemptToFixPassed + }) => { const store = storage('legacy').getStore() const span = store?.span @@ -208,6 +216,12 @@ class MochaPlugin extends CiPlugin { if (hasBeenRetried) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( @@ -403,18 +417,18 @@ class MochaPlugin extends CiPlugin { startTestSpan (testInfo) { const { + testName, testSuiteAbsolutePath, title, isNew, isEfdRetry, testStartLine, isParallel, + isAttemptToFix, isDisabled, isQuarantined } = testInfo - const testName = removeEfdStringFromTestName(testInfo.testName) - const extraTags = {} const testParametersString = getTestParametersString(this._testTitleToParams, title) if (testParametersString) { @@ -429,6 +443,12 @@ class MochaPlugin extends CiPlugin { extraTags[MOCHA_IS_PARALLEL] = 'true' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } + if (isDisabled) { extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' } From e4254e655b6d319bff410371c23355653399e363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Thu, 20 Mar 2025 15:11:49 +0100 Subject: [PATCH 12/16] Add support for Playwright --- .../attempt-to-fix-test.js | 13 ++ .../playwright/playwright.spec.js | 137 +++++++++++++++++- .../src/playwright.js | 123 ++++++++++++---- .../datadog-plugin-playwright/src/index.js | 22 ++- 4 files changed, 267 insertions(+), 28 deletions(-) create mode 100644 integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js diff --git a/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js new file mode 100644 index 00000000000..a4ed9aae722 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js @@ -0,0 +1,13 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('attempt to fix', () => { + test('should attempt to fix failed test', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello Warld' + ]) + }) +}) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 7dfb3ee3fe6..61cc461a383 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -33,7 +33,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_NAME } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -895,6 +898,138 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptingToFix) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix should attempt to fix failed test' + ) + + // Retried tests are in "randomly" order + let countAttemptToFixTests = 0 + let hasFailedAllRetries = false + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (isAttemptingToFix && + test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true' && + test.meta[TEST_IS_RETRY] === 'true' && + test.meta[TEST_RETRY_REASON] === 'attempt_to_fix') { + countAttemptToFixTests++ + if (test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true') { + hasFailedAllRetries = true + } + } + } + + if (isAttemptingToFix) { + assert.equal(countAttemptToFixTests, retriedTests.length - 1) + assert.equal(hasFailedAllRetries, true) + } else { + assert.equal(countAttemptToFixTests, 0) + assert.equal(hasFailedAllRetries, false) + } + }) + + const runAttemptToFixTest = (done, isAttemptingToFix, isQuarantined, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isAttemptingToFix) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js attempt-to-fix-test.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-test-management', + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isAttemptingToFix && isQuarantined) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, true, false) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, true, true) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index f56f88a565f..683ad5a7abd 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -43,7 +43,9 @@ let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 let knownTests = {} let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} +const testsAttemptToFix = [] let rootDir = '' const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' @@ -51,10 +53,9 @@ function getTestProperties (test) { const testName = getTestFullname(test) const testSuite = getTestSuitePath(test._requireFile, rootDir) - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.playwright?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function isNewTest (test) { @@ -73,18 +74,24 @@ function getSuiteType (test, type) { } // Copy of Suite#_deepClone but with a function to filter tests -function deepCloneSuite (suite, filterTest) { +function deepCloneSuite (suite, retryNumber, filterTestAttemptToFix, filterTestEfd) { const copy = suite._clone() for (const entry of suite._entries) { if (entry.constructor.name === 'Suite') { - copy._addSuite(deepCloneSuite(entry, filterTest)) + copy._addSuite(deepCloneSuite(entry, retryNumber, filterTestAttemptToFix, filterTestEfd)) } else { - if (filterTest(entry)) { - const copiedTest = entry._clone() + const copiedTest = entry._clone() + + if (filterTestAttemptToFix && filterTestAttemptToFix(entry)) { + if (retryNumber === testManagementAttemptToFixRetries) { + copiedTest._ddIsLastRetry = true + } + copiedTest._ddIsAttemptToFix = true + } else if (filterTestEfd && filterTestEfd(entry)) { copiedTest._ddIsNew = true copiedTest._ddIsEfdRetry = true - copy._addTest(copiedTest) } + copy._addTest(copiedTest) } } return copy @@ -276,6 +283,11 @@ function testBeginHandler (test, browserName) { }) } + // We disable retries by default if attemptToFix is true + if (getTestProperties(test).attemptToFix) { + test.retries = 0 + } + const testAsyncResource = new AsyncResource('bound-anonymous-fn') testToAr.set(test, testAsyncResource) testAsyncResource.runInAsyncScope(() => { @@ -305,6 +317,31 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { return } + if (testSuiteToTestStatuses.has(testSuiteAbsolutePath)) { + const testStatusMap = testSuiteToTestStatuses.get(testSuiteAbsolutePath) + const testFullName = getTestFullname(test) + if (testStatusMap.has(testFullName)) { + testStatusMap.get(testFullName).push(testStatus) + } else { + testStatusMap.set(testFullName, [testStatus]) + } + } else { + const testStatusMap = new Map() + testStatusMap.set(getTestFullname(test), [testStatus]) + testSuiteToTestStatuses.set(testSuiteAbsolutePath, testStatusMap) + } + + let hasFailedAllRetries = false + let hasPassedAttemptToFixRetries = false + if (test._ddIsLastRetry) { + const statuses = testSuiteToTestStatuses.get(testSuiteAbsolutePath).get(getTestFullname(test)) + if (statuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (statuses.every(status => status === 'pass')) { + hasPassedAttemptToFixRetries = true + } + } + const testResult = results[results.length - 1] const testAsyncResource = testToAr.get(test) testAsyncResource.runInAsyncScope(() => { @@ -315,17 +352,14 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { error, extraTags: annotationTags, isNew: test._ddIsNew, + isAttemptToFix: test._ddIsAttemptToFix, isQuarantined: test._ddIsQuarantined, - isEfdRetry: test._ddIsEfdRetry + isEfdRetry: test._ddIsEfdRetry, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) }) - if (testSuiteToTestStatuses.has(testSuiteAbsolutePath)) { - testSuiteToTestStatuses.get(testSuiteAbsolutePath).push(testStatus) - } else { - testSuiteToTestStatuses.set(testSuiteAbsolutePath, [testStatus]) - } - if (error) { addErrorToTestSuite(testSuiteAbsolutePath, error) } @@ -338,11 +372,11 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { // Last test, we finish the suite if (!remainingTestsByFile[testSuiteAbsolutePath].length) { const testStatuses = testSuiteToTestStatuses.get(testSuiteAbsolutePath) - + const testStatusesFlat = Array.from(testStatuses.values()).flat() let testSuiteStatus = 'pass' - if (testStatuses.some(status => status === 'fail')) { + if (testStatusesFlat.some(status => status === 'fail')) { testSuiteStatus = 'fail' - } else if (testStatuses.every(status => status === 'skip')) { + } else if (testStatusesFlat.every(status => status === 'skip')) { testSuiteStatus = 'skip' } @@ -445,6 +479,7 @@ function runnerHook (runnerExport, playwrightVersion) { isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isEarlyFlakeDetectionEnabled = false @@ -485,7 +520,10 @@ function runnerHook (runnerExport, playwrightVersion) { const projects = getProjectsFromRunner(this) - if (isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0) { + const shouldSetRetries = isFlakyTestRetriesEnabled && + flakyTestRetriesCount > 0 && + !isTestManagementTestsEnabled + if (shouldSetRetries) { projects.forEach(project => { if (project.retries === 0) { // Only if it hasn't been set by the user project.retries = flakyTestRetriesCount @@ -493,7 +531,7 @@ function runnerHook (runnerExport, playwrightVersion) { }) } - const runAllTestsReturn = await runAllTests.apply(this, arguments) + let runAllTestsReturn = await runAllTests.apply(this, arguments) Object.values(remainingTestsByFile).forEach(tests => { // `tests` should normally be empty, but if it isn't, @@ -508,6 +546,27 @@ function runnerHook (runnerExport, playwrightVersion) { const sessionStatus = runAllTestsReturn.status || runAllTestsReturn + if (isTestManagementTestsEnabled && sessionStatus === 'failed') { + let failedTestsCount = 0 + testSuiteToTestStatuses.forEach(testStatuses => { + testStatuses.forEach(statuses => { + failedTestsCount += statuses.filter(status => status === 'fail').length + }) + }) + for (const test of testsAttemptToFix) { + const fullname = getTestFullname(test) + const testProperties = getTestProperties(test) + if (testProperties.quarantined || testProperties.disabled) { + const testStatuses = testSuiteToTestStatuses.get(test.parent._requireFile).get(fullname) + failedTestsCount -= testStatuses.filter(status => status === 'fail').length + } + } + + if (failedTestsCount === 0) { + runAllTestsReturn = 'passed' + } + } + const flushWait = new Promise(resolve => { onDone = resolve }) @@ -606,7 +665,17 @@ addHook({ if (isTestManagementTestsEnabled) { for (const test of allTests) { const testProperties = getTestProperties(test) - if (testProperties.disabled) { + if (testProperties.attemptToFix) { + const fileSuite = getSuiteType(test, 'file') + const projectSuite = getSuiteType(test, 'project') + const isAttemptToFix = test => getTestProperties(test).attemptToFix + for (let repeatEachIndex = 1; repeatEachIndex <= testManagementAttemptToFixRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, repeatEachIndex, isAttemptToFix, null) + applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) + projectSuite._addSuite(copyFileSuite) + } + testsAttemptToFix.push(test) + } else if (testProperties.disabled) { test._ddIsDisabled = true test.expectedStatus = 'skipped' } else if (testProperties.quarantined) { @@ -619,18 +688,22 @@ addHook({ if (isKnownTestsEnabled) { const newTests = allTests.filter(isNewTest) - newTests.forEach(newTest => { + for (const newTest of newTests) { + // If the test is an attempt to fix, we don't need to apply EFD + if (getTestProperties(newTest).attemptToFix) { + continue + } newTest._ddIsNew = true if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { const fileSuite = getSuiteType(newTest, 'file') const projectSuite = getSuiteType(newTest, 'project') - for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { - const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) + for (let repeatEachIndex = 1; repeatEachIndex <= earlyFlakeDetectionNumRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, repeatEachIndex, null, isNewTest) applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) projectSuite._addSuite(copyFileSuite) } } - }) + } } return rootSuite diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 9c75c690bc4..03dc1f1e20b 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -20,7 +20,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, TEST_BROWSER_NAME, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -141,6 +144,7 @@ class PlaywrightPlugin extends CiPlugin { testSuiteAbsolutePath, testSourceLine, browserName, + isAttemptToFix, isDisabled }) => { const store = storage('legacy').getStore() @@ -162,7 +166,10 @@ class PlaywrightPlugin extends CiPlugin { isNew, isEfdRetry, isRetry, - isQuarantined + isAttemptToFix, + isQuarantined, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) => { const store = storage('legacy').getStore() const span = store && store.span @@ -186,6 +193,17 @@ class PlaywrightPlugin extends CiPlugin { if (isRetry) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (isAttemptToFix) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + if (hasPassedAttemptToFixRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } if (isQuarantined) { span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') } From 626a509fb8eb3fc7e0d882cac2a1f01853ff0ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Thu, 20 Mar 2025 17:57:27 +0100 Subject: [PATCH 13/16] Add support for Vitest --- .../vitest-tests/test-attempt-to-fix.mjs | 9 ++ integration-tests/vitest/vitest.spec.js | 142 +++++++++++++++++- .../datadog-instrumentations/src/vitest.js | 83 ++++++++-- packages/datadog-plugin-vitest/src/index.js | 47 +++++- 4 files changed, 258 insertions(+), 23 deletions(-) create mode 100644 integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs diff --git a/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs new file mode 100644 index 00000000000..f2d480a8e3b --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs @@ -0,0 +1,9 @@ +import { describe, test, expect } from 'vitest' + +describe('attempt to fix tests', () => { + test('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running') // to check if this is being run + expect(1 + 2).to.equal(4) + }) +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index a4d68ce87b8..205980f9a37 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -37,7 +37,9 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -1341,6 +1343,144 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptingToFix) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs.attempt to fix tests can attempt to fix a test' + ] + ) + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (isAttemptingToFix && i !== 0) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + + if (isAttemptingToFix && i === retriedTests.length - 1) { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + }) + + const runAttemptToFixTest = (done, isAttemptingToFix, isQuarantining, extraEnvVars = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions(isAttemptingToFix) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/test-attempt-to-fix*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init --no-warnings', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running') + if (isAttemptingToFix && isQuarantining) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, true, false) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, true, true) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index 4ac4062b46c..fe729897195 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -9,6 +9,7 @@ const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') const isNewTestCh = channel('ci:vitest:test:is-new') +const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix') const isDisabledCh = channel('ci:vitest:test:is-disabled') const isQuarantinedCh = channel('ci:vitest:test:is-quarantined') @@ -30,7 +31,9 @@ const taskToStatuses = new WeakMap() const newTasks = new WeakSet() const disabledTasks = new WeakSet() const quarantinedTasks = new WeakSet() +const attemptToFixTasks = new WeakSet() let isRetryReasonEfd = false +let isRetryReasonAttemptToFix = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -53,6 +56,7 @@ function getProvidedContext () { _ddEarlyFlakeDetectionNumRetries: numRepeats, _ddIsKnownTestsEnabled: isKnownTestsEnabled, _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled, + _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries, _ddTestManagementTests: testManagementTests, _ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled } = globalThis.__vitest_worker__.providedContext @@ -64,6 +68,7 @@ function getProvidedContext () { numRepeats, isKnownTestsEnabled, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests, isFlakyTestRetriesEnabled } @@ -76,6 +81,7 @@ function getProvidedContext () { numRepeats: 0, isKnownTestsEnabled: false, isTestManagementTestsEnabled: false, + testManagementAttemptToFixRetries: 0, testManagementTests: {} } } @@ -176,6 +182,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionFaulty = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false + let testManagementAttemptToFixRetries = 0 let isDiEnabled = false let knownTests = {} let testManagementTests = {} @@ -190,6 +197,7 @@ function getSortWrapper (sort) { isDiEnabled = libraryConfig.isDiEnabled isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isFlakyTestRetriesEnabled = false @@ -262,6 +270,7 @@ function getSortWrapper (sort) { try { const workspaceProject = this.ctx.getCoreWorkspaceProject() workspaceProject._provided._ddIsTestManagementTestsEnabled = isTestManagementTestsEnabled + workspaceProject._provided._ddTestManagementAttemptToFixRetries = testManagementAttemptToFixRetries workspaceProject._provided._ddTestManagementTests = testManagementTests } catch (e) { log.warn('Could not send test management tests to workers so Test Management will not work.') @@ -353,10 +362,24 @@ addHook({ isKnownTestsEnabled, numRepeats, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests } = getProvidedContext() if (isTestManagementTestsEnabled) { + isAttemptToFixCh.publish({ + testManagementTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isAttemptToFix) => { + if (isAttemptToFix) { + isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries + task.repeats = testManagementAttemptToFixRetries + attemptToFixTasks.add(task) + taskToStatuses.set(task, []) + } + } + }) isDisabledCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, @@ -364,7 +387,9 @@ addHook({ onDone: (isTestDisabled) => { if (isTestDisabled) { disabledTasks.add(task) - task.mode = 'skip' + if (!attemptToFixTasks.has(task)) { + task.mode = 'skip' + } } } }) @@ -376,7 +401,7 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { - if (isNew) { + if (isNew && !attemptToFixTasks.has(task)) { if (isEarlyFlakeDetectionEnabled) { isRetryReasonEfd = task.repeats !== numRepeats task.repeats = numRepeats @@ -396,7 +421,20 @@ addHook({ shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) { const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { + if (isTestManagementTestsEnabled) { + if (attemptToFixTasks.has(task)) { + if (disabledTasks.has(task) || quarantinedTasks.has(task)) { + if (task.result.state === 'fail') { + switchedStatuses.add(task) + } + task.result.state = 'pass' + } + } else if (quarantinedTasks.has(task)) { + task.result.state = 'pass' + } + } + + if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) { const statuses = taskToStatuses.get(task) // If the test has passed at least once, we consider it passed if (statuses.includes('pass')) { @@ -407,12 +445,6 @@ addHook({ } } - if (isTestManagementTestsEnabled) { - if (quarantinedTasks.has(task)) { - task.result.state = 'pass' - } - } - return onAfterRunTask.apply(this, arguments) }) @@ -424,6 +456,7 @@ addHook({ } const testName = getTestName(task) let isNew = false + const isAttemptToFix = attemptToFixTasks.has(task) let isQuarantined = false const { @@ -444,7 +477,9 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isTestQuarantined) => { - isQuarantined = isTestQuarantined + if (!isAttemptToFix) { + isQuarantined = isTestQuarantined + } if (isTestQuarantined) { quarantinedTasks.add(task) } @@ -484,7 +519,7 @@ addHook({ // These clauses handle task.repeats, whether EFD is enabled or not // The only thing that EFD does is to forcefully pass the test if it has passed at least once - if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed + if (numRepetition > 0 && numRepetition <= task.repeats) { // it may or may have not failed // Here we finish the earlier iteration, // as long as it's not the _last_ iteration (which will be finished normally) @@ -501,7 +536,7 @@ addHook({ testPassCh.publish({ task }) }) } - if (isEarlyFlakeDetectionEnabled) { + if (isEarlyFlakeDetectionEnabled || attemptToFixTasks.has(task)) { const statuses = taskToStatuses.get(task) statuses.push(lastExecutionStatus) // If we don't "reset" the result.state to "pass", once a repetition fails, @@ -532,8 +567,10 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, isRetryReasonEfd, + isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0, isNew, mightHitProbe: isDiEnabled && numAttempt > 0, + isAttemptToFix: isAttemptToFix && numRepetition > 0, isQuarantined }) }) @@ -548,6 +585,8 @@ addHook({ } const result = await onAfterTryTask.apply(this, arguments) + const { testManagementAttemptToFixRetries } = getProvidedContext() + const status = getVitestTestStatus(task, retryCount) const asyncResource = taskToAsync.get(task) @@ -557,10 +596,18 @@ addHook({ await waitForHitProbe() } + let attemptToFixPassed = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.length === testManagementAttemptToFixRetries && statuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + if (asyncResource) { // We don't finish here because the test might fail in a later hook (afterEach) asyncResource.runInAsyncScope(() => { - testFinishTimeCh.publish({ status, task }) + testFinishTimeCh.publish({ status, task, attemptToFixPassed }) }) } @@ -715,11 +762,19 @@ addHook({ testError = errors[0] } + let hasFailedAllRetries = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } + } + if (testAsyncResource) { const isRetry = task.result?.retryCount > 0 // `duration` is the duration of all the retries, so it can't be used if there are retries testAsyncResource.runInAsyncScope(() => { - testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError }) + testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError, hasFailedAllRetries }) }) } if (errors?.length) { diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 06a5257d905..a795685e93c 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -23,7 +23,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -53,18 +56,30 @@ class VitestPlugin extends CiPlugin { onDone(!testsForThisTestSuite.includes(testName)) }) + this.addSub('ci:vitest:test:is-attempt-to-fix', ({ + testManagementTests, + testSuiteAbsolutePath, + testName, + onDone + }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const { isAttemptToFix } = this.getTestProperties(testManagementTests, testSuite, testName) + + onDone(isAttemptToFix ?? false) + }) + this.addSub('ci:vitest:test:is-disabled', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isDisabled } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isDisabled ?? false) + onDone(isDisabled) }) this.addSub('ci:vitest:test:is-quarantined', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isQuarantined } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isQuarantined ?? false) + onDone(isQuarantined) }) this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ @@ -85,9 +100,11 @@ class VitestPlugin extends CiPlugin { testSuiteAbsolutePath, isRetry, isNew, + isAttemptToFix, isQuarantined, mightHitProbe, - isRetryReasonEfd + isRetryReasonEfd, + isRetryReasonAttemptToFix }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage('legacy').getStore() @@ -104,6 +121,12 @@ class VitestPlugin extends CiPlugin { if (isRetryReasonEfd) { extraTags[TEST_RETRY_REASON] = 'efd' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + if (isRetryReasonAttemptToFix) { + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } if (isQuarantined) { extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' } @@ -124,7 +147,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + this.addSub('ci:vitest:test:finish-time', ({ status, task, attemptToFixPassed }) => { const store = storage('legacy').getStore() const span = store?.span @@ -132,6 +155,11 @@ class VitestPlugin extends CiPlugin { // this is because the test might fail at a `afterEach` hook if (span) { span.setTag(TEST_STATUS, status) + + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + this.taskToFinishTime.set(task, span._getTime()) } }) @@ -150,7 +178,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises }) => { + this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises, hasFailedAllRetries }) => { const store = storage('legacy').getStore() const span = store?.span @@ -172,6 +200,9 @@ class VitestPlugin extends CiPlugin { if (error) { span.setTag('error', error) } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } if (duration) { span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds } else { @@ -327,10 +358,10 @@ class VitestPlugin extends CiPlugin { } getTestProperties (testManagementTests, testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.vitest?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } } From 6a08c602481a0ae75306755aa7bfccc164be668e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Fri, 21 Mar 2025 13:05:47 +0100 Subject: [PATCH 14/16] Add cypress support --- integration-tests/cypress/cypress.spec.js | 119 +++++++++++++++++- .../cypress/e2e/attempt-to-fix.js | 8 ++ .../src/cypress-plugin.js | 84 ++++++++++--- .../datadog-plugin-cypress/src/support.js | 54 ++++++-- 4 files changed, 240 insertions(+), 25 deletions(-) create mode 100644 integration-tests/cypress/e2e/attempt-to-fix.js diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 76958fbdac0..04775d01fc0 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -44,7 +44,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_NAME, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1775,6 +1778,120 @@ moduleTypes.forEach(({ }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isAttemptToFix) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'cypress/e2e/attempt-to-fix.js.attempt to fix is attempt to fix' + ] + ) + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix is attempt to fix' + ) + + // Events come in reverse order + for (let i = retriedTests.length - 1; i >= 0; i--) { + const test = retriedTests[i] + if (isAttemptToFix && i !== retriedTests.length - 1) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + + if (isAttemptToFix && i === 0) { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + }) + + const runAttemptToFixTest = (done, isAttemptToFix, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isAttemptToFix) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/attempt-to-fix.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + } + + it('can attempt to fix tests', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, true) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/cypress/e2e/attempt-to-fix.js b/integration-tests/cypress/e2e/attempt-to-fix.js new file mode 100644 index 00000000000..2cd64a6c8cb --- /dev/null +++ b/integration-tests/cypress/e2e/attempt-to-fix.js @@ -0,0 +1,8 @@ +/* eslint-disable */ +describe('attempt to fix', () => { + it('is attempt to fix', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello Warld') + }) +}) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index de9e56f5c7a..431b95ab41f 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -39,7 +39,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS + DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -223,6 +226,7 @@ class CypressPlugin { commitMessage } this.finishedTestsByFile = {} + this.testStatuses = {} this.isTestsSkipped = false this.isSuitesSkippingEnabled = false @@ -236,6 +240,8 @@ class CypressPlugin { this.hasUnskippableSuites = false this.unskippableSuites = [] this.knownTests = [] + this.isTestManagementTestsEnabled = false + this.testManagementAttemptToFixRetries = 0 } // Init function returns a promise that resolves with the Cypress configuration @@ -264,7 +270,8 @@ class CypressPlugin { isFlakyTestRetriesEnabled, flakyTestRetriesCount, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled @@ -276,6 +283,7 @@ class CypressPlugin { this.cypressConfig.retries.runMode = flakyTestRetriesCount } this.isTestManagementTestsEnabled = isTestManagementEnabled + this.testManagementAttemptToFixRetries = testManagementAttemptToFixRetries } return this.cypressConfig }) @@ -283,10 +291,10 @@ class CypressPlugin { } getTestProperties (testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = this.testManagementTests?.cypress?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) { @@ -431,6 +439,24 @@ class CypressPlugin { this.isTestManagementTestsEnabled = false } else { this.testManagementTests = testManagementTestsResponse.testManagementTests + // TODO - REMOVE AFTERWARDS + // this.testManagementTests = { + // cypress: { + // suites: { + // 'cypress/e2e/spec.cy.js': { + // tests: { + // 'My First Test Gets, types and asserts': { + // properties: { + // is_attempt_to_fix: true, + // disabled: false, + // quarantined: true + // } + // } + // } + // } + // } + // } + // } } } @@ -599,12 +625,14 @@ class CypressPlugin { skippedTestSpan.setTag(ITR_CORRELATION_ID, this.itrCorrelationId) } - const { isDisabled, isQuarantined } = this.getTestProperties(spec.relative, cypressTestName) + const { isAttemptToFix, isDisabled, isQuarantined } = this.getTestProperties(spec.relative, cypressTestName) - if (isDisabled) { - skippedTestSpan.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true') - } else if (isQuarantined) { - skippedTestSpan.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + if (!isAttemptToFix) { + if (isDisabled) { + skippedTestSpan.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true') + } else if (isQuarantined) { + skippedTestSpan.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } } skippedTestSpan.finish() @@ -694,7 +722,10 @@ class CypressPlugin { isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries, - isKnownTestsEnabled: this.isKnownTestsEnabled + isKnownTestsEnabled: this.isKnownTestsEnabled, + isTestManagementEnabled: this.isTestManagementTestsEnabled, + testManagementAttemptToFixRetries: this.testManagementAttemptToFixRetries, + testManagementTests: this.testManagementTests } if (this.testSuiteSpan) { @@ -710,8 +741,7 @@ class CypressPlugin { }) const isUnskippable = this.unskippableSuites.includes(testSuite) const isForcedToRun = shouldSkip && isUnskippable - const { isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) - + const { isAttemptToFix, isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) // skip test if (shouldSkip && !isUnskippable) { this.skippedTests.push(test) @@ -721,7 +751,7 @@ class CypressPlugin { // TODO: I haven't found a way to trick cypress into ignoring a test // The way we'll implement quarantine in cypress is by skipping the test altogether - if (isDisabled || isQuarantined) { + if (!isAttemptToFix && (isDisabled || isQuarantined)) { return { shouldSkip: true } } @@ -750,7 +780,9 @@ class CypressPlugin { testSuiteAbsolutePath, testName, isNew, - isEfdRetry + isEfdRetry, + isAttemptToFix, + isLastRetry } = test if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) @@ -773,6 +805,13 @@ class CypressPlugin { const testStatus = CYPRESS_STATUS_TO_TEST_STATUS[state] this.activeTestSpan.setTag(TEST_STATUS, testStatus) + // Save the test status to know if it has passed all retries + if (!this.testStatuses[testName]) { + this.testStatuses[testName] = [testStatus] + } else { + this.testStatuses[testName].push(testStatus) + } + if (error) { this.activeTestSpan.setTag('error', error) } @@ -789,6 +828,23 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } + if (isAttemptToFix) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + this.activeTestSpan.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + if (isLastRetry) { + if (this.testStatuses[testName].every(status => status === 'pass')) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + } + + if (isLastRetry) { + if (this.testStatuses[testName].every(status => status === 'fail')) { + this.activeTestSpan.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + const finishedTest = { testName, testStatus, diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 749a25d7f66..cd2ec08083c 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -4,6 +4,9 @@ let isKnownTestsEnabled = false let knownTestsForSuite = [] let suiteTests = [] let earlyFlakeDetectionNumRetries = 0 +let isTestManagementEnabled = false +let testManagementAttemptToFixRetries = 0 +let testManagementTests = {} // We need to grab the original window as soon as possible, // in case the test changes the origin. If the test does change the origin, // any call to `cy.window()` will result in a cross origin error. @@ -23,31 +26,57 @@ function isNewTest (test) { return !knownTestsForSuite.includes(test.fullTitle()) } -function retryTest (test, suiteTests) { - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { +function getTestProperties (testSuite, testName) { + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = + testManagementTests?.cypress?.suites?.[testSuite]?.tests?.[testName]?.properties || {} + + return { isAttemptToFix, isDisabled, isQuarantined } +} + +function retryTest (test, suiteTests, numRetries, isAttemptToFix) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() // TODO: signal in framework logs that this is a retry. // TODO: Change it so these tests are allowed to fail. // TODO: figure out if reported duration is skewed. suiteTests.unshift(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + if (isAttemptToFix) { + clonedTest._ddIsAttemptToFix = true + if (retryIndex === numRetries - 1) { + clonedTest._ddIsLastRetry = true + } + } else { + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } } } const oldRunTests = Cypress.mocha.getRunner().runTests Cypress.mocha.getRunner().runTests = function (suite, fn) { - if (!isKnownTestsEnabled) { + if (!isKnownTestsEnabled && !isTestManagementEnabled) { return oldRunTests.apply(this, arguments) } // We copy the new tests at the beginning of the suite run (runTests), so that they're run // multiple times. suite.tests.forEach(test => { - if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { - test._ddIsNew = true - if (isEarlyFlakeDetectionEnabled) { - retryTest(test, suite.tests) + const testName = test.fullTitle() + const testSuite = Cypress.mocha.getRootSuite().file + + const { isAttemptToFix } = getTestProperties(testSuite, testName) + + if (isTestManagementEnabled) { + if (isAttemptToFix) { + retryTest(test, suite.tests, testManagementAttemptToFixRetries, true) + } + } + if (isKnownTestsEnabled) { + if (!test._ddIsNew && !test.isPending() && isNewTest(test) && !isAttemptToFix) { + test._ddIsNew = true + if (isEarlyFlakeDetectionEnabled) { + retryTest(test, suite.tests, earlyFlakeDetectionNumRetries, false) + } } } }) @@ -80,6 +109,9 @@ before(function () { isKnownTestsEnabled = suiteConfig.isKnownTestsEnabled knownTestsForSuite = suiteConfig.knownTestsForSuite earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries + isTestManagementEnabled = suiteConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = suiteConfig.testManagementAttemptToFixRetries + testManagementTests = suiteConfig.testManagementTests } }) }) @@ -104,7 +136,9 @@ afterEach(function () { state: currentTest.state, error: currentTest.err, isNew: currentTest._ddIsNew, - isEfdRetry: currentTest._ddIsEfdRetry + isEfdRetry: currentTest._ddIsEfdRetry, + isAttemptToFix: currentTest._ddIsAttemptToFix, + isLastRetry: currentTest._ddIsLastRetry } try { testInfo.testSourceLine = Cypress.mocha.getRunner().currentRunnable.invocationDetails.line From 5b8f2d1b6eddfe5cea7eea2dfa45f9338a3f02d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Fri, 21 Mar 2025 13:29:44 +0100 Subject: [PATCH 15/16] Remove unused code --- .../src/cypress-plugin.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 431b95ab41f..759a2b66ece 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -439,24 +439,6 @@ class CypressPlugin { this.isTestManagementTestsEnabled = false } else { this.testManagementTests = testManagementTestsResponse.testManagementTests - // TODO - REMOVE AFTERWARDS - // this.testManagementTests = { - // cypress: { - // suites: { - // 'cypress/e2e/spec.cy.js': { - // tests: { - // 'My First Test Gets, types and asserts': { - // properties: { - // is_attempt_to_fix: true, - // disabled: false, - // quarantined: true - // } - // } - // } - // } - // } - // } - // } } } From 7a2740736bbf79e857fa2548782ff5674c5535d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Fri, 21 Mar 2025 14:02:35 +0100 Subject: [PATCH 16/16] Fix compatibility of cypress --- packages/datadog-plugin-cypress/src/support.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index cd2ec08083c..d103d4c48aa 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -27,10 +27,18 @@ function isNewTest (test) { } function getTestProperties (testSuite, testName) { - const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = - testManagementTests?.cypress?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - - return { isAttemptToFix, isDisabled, isQuarantined } + // We neeed to do it in this way because of compatibility with older versions as '?' is not supported in older versions of Cypress + const properties = testManagementTests && + testManagementTests.cypress && + testManagementTests.cypress.suites && + testManagementTests.cypress.suites[testSuite] && + testManagementTests.cypress.suites[testSuite].tests && + testManagementTests.cypress.suites[testSuite].tests[testName] && + testManagementTests.cypress.suites[testSuite].tests[testName].properties || {}; + + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = properties; + + return { isAttemptToFix, isDisabled, isQuarantined }; } function retryTest (test, suiteTests, numRetries, isAttemptToFix) {