Skip to content

Commit cc8ac7a

Browse files
MariovidoMariovido
committed
Add cucumber support
1 parent 2740ada commit cc8ac7a

File tree

4 files changed

+240
-28
lines changed

4 files changed

+240
-28
lines changed

‎integration-tests/ci-visibility/features-test-management/support/steps.js

+7
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,10 @@ When('the greeter says disabled', function () {
2121
// expected to fail if not disabled
2222
this.whatIHeard = 'disabld'
2323
})
24+
25+
When('the greeter says attempt to fix', function () {
26+
// eslint-disable-next-line no-console
27+
console.log('I am running') // just to assert whether this is running
28+
// expected to fail
29+
this.whatIHeard = 'attempt to fx'
30+
})

‎integration-tests/cucumber/cucumber.spec.js

+136-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@ const {
4949
TEST_MANAGEMENT_IS_DISABLED,
5050
DD_CAPABILITIES_TEST_IMPACT_ANALYSIS,
5151
DD_CAPABILITIES_EARLY_FLAKE_DETECTION,
52-
DD_CAPABILITIES_AUTO_TEST_RETRIES
52+
DD_CAPABILITIES_AUTO_TEST_RETRIES,
53+
TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
54+
TEST_HAS_FAILED_ALL_RETRIES
5355
} = require('../../packages/dd-trace/src/plugins/util/test')
5456
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
5557

@@ -2031,6 +2033,139 @@ versions.forEach(version => {
20312033
})
20322034

20332035
context('test management', () => {
2036+
context('attempt to fix', () => {
2037+
beforeEach(() => {
2038+
receiver.setTestManagementTests({
2039+
cucumber: {
2040+
suites: {
2041+
'ci-visibility/features-test-management/attempt-to-fix.feature': {
2042+
tests: {
2043+
'Say attempt to fix': {
2044+
properties: {
2045+
attempt_to_fix: true
2046+
}
2047+
}
2048+
}
2049+
}
2050+
}
2051+
}
2052+
})
2053+
})
2054+
2055+
const getTestAssertions = (isAttemptToFix) =>
2056+
receiver
2057+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2058+
const events = payloads.flatMap(({ payload }) => payload.events)
2059+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2060+
const testSession = events.find(event => event.type === 'test_session_end').content
2061+
2062+
if (isAttemptToFix) {
2063+
assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
2064+
} else {
2065+
assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
2066+
}
2067+
2068+
const retriedTests = tests.filter(
2069+
test => test.meta[TEST_NAME] === 'Say attempt to fix'
2070+
)
2071+
2072+
for (let i = 0; i < retriedTests.length; i++) {
2073+
const test = retriedTests[i]
2074+
2075+
const testResource = 'ci-visibility/features-test-management/attempt-to-fix.feature.' +
2076+
'Say attempt to fix'
2077+
assert.equal(test.resource, testResource)
2078+
2079+
if (isAttemptToFix && i !== 0) {
2080+
assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
2081+
assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
2082+
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
2083+
} else {
2084+
assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX)
2085+
assert.notProperty(test.meta, TEST_IS_RETRY)
2086+
assert.notProperty(test.meta, TEST_RETRY_REASON)
2087+
}
2088+
2089+
if (isAttemptToFix && i === retriedTests.length - 1) {
2090+
assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true')
2091+
}
2092+
}
2093+
})
2094+
2095+
const runTest = (done, isAttemptToFix, isQuarantined, extraEnvVars) => {
2096+
const testAssertionsPromise = getTestAssertions(isAttemptToFix)
2097+
let stdout = ''
2098+
2099+
childProcess = exec(
2100+
'./node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature',
2101+
{
2102+
cwd,
2103+
env: {
2104+
...getCiVisAgentlessConfig(receiver.port),
2105+
...extraEnvVars
2106+
},
2107+
stdio: 'inherit'
2108+
}
2109+
)
2110+
2111+
childProcess.stdout.on('data', (data) => {
2112+
stdout += data.toString()
2113+
})
2114+
2115+
childProcess.on('exit', exitCode => {
2116+
testAssertionsPromise.then(() => {
2117+
assert.include(stdout, 'I am running')
2118+
if (isQuarantined) {
2119+
assert.equal(exitCode, 0)
2120+
} else {
2121+
assert.equal(exitCode, 1)
2122+
}
2123+
done()
2124+
}).catch(done)
2125+
})
2126+
}
2127+
2128+
it('can attempt to fix tests', (done) => {
2129+
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
2130+
2131+
runTest(done, true, false)
2132+
})
2133+
2134+
it('does not attempt to fix tests if test management is not enabled', (done) => {
2135+
receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } })
2136+
2137+
runTest(done, false, false)
2138+
})
2139+
2140+
it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
2141+
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
2142+
2143+
runTest(done, false, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
2144+
})
2145+
2146+
it('does not fail retry if a test is quarantined', (done) => {
2147+
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
2148+
receiver.setTestManagementTests({
2149+
cucumber: {
2150+
suites: {
2151+
'ci-visibility/features-test-management/attempt-to-fix.feature': {
2152+
tests: {
2153+
'Say attempt to fix': {
2154+
properties: {
2155+
attempt_to_fix: true,
2156+
quarantined: true
2157+
}
2158+
}
2159+
}
2160+
}
2161+
}
2162+
}
2163+
})
2164+
2165+
runTest(done, true, true)
2166+
})
2167+
})
2168+
20342169
context('disabled', () => {
20352170
beforeEach(() => {
20362171
receiver.setTestManagementTests({

‎packages/datadog-instrumentations/src/cucumber.js

+77-26
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ let isEarlyFlakeDetectionFaulty = false
7373
let isFlakyTestRetriesEnabled = false
7474
let isKnownTestsEnabled = false
7575
let isTestManagementTestsEnabled = false
76+
let testManagementAttemptToFixRetries = 0
7677
let testManagementTests = {}
7778
let numTestRetries = 0
7879
let knownTests = []
@@ -121,10 +122,10 @@ function isNewTest (testSuite, testName) {
121122
}
122123

123124
function getTestProperties (testSuite, testName) {
124-
const { disabled, quarantined } =
125+
const { attempt_to_fix: attemptToFix, disabled, quarantined } =
125126
testManagementTests?.cucumber?.suites?.[testSuite]?.tests?.[testName]?.properties || {}
126127

127-
return { disabled, quarantined }
128+
return { attemptToFix, disabled, quarantined }
128129
}
129130

130131
function getTestStatusFromRetries (testStatuses) {
@@ -303,22 +304,46 @@ function wrapRun (pl, isLatestVersion) {
303304
}
304305
let isNew = false
305306
let isEfdRetry = false
307+
let isAttemptToFix = false
308+
let isAttemptToFixRetry = false
309+
let hasFailedAllRetries = false
310+
let hasPassedAllRetries = false
306311
let isDisabled = false
307312
let isQuarantined = false
308-
if (isKnownTestsEnabled && status !== 'skip') {
309-
const numRetries = numRetriesByPickleId.get(this.pickle.id)
310-
311-
isNew = numRetries !== undefined
312-
isEfdRetry = numRetries > 0
313-
}
314313
if (isTestManagementTestsEnabled) {
315314
const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd())
316315
const testProperties = getTestProperties(testSuitePath, this.pickle.name)
317-
isDisabled = testProperties.disabled
318-
if (!isDisabled) {
319-
isQuarantined = testProperties.quarantined
316+
const numRetries = numRetriesByPickleId.get(this.pickle.id)
317+
isAttemptToFix = testProperties.attemptToFix
318+
isAttemptToFixRetry = isAttemptToFix && numRetries > 0
319+
if (!isAttemptToFix) {
320+
isDisabled = testProperties.disabled
321+
if (!isDisabled) {
322+
isQuarantined = testProperties.quarantined
323+
}
324+
}
325+
326+
if (isAttemptToFixRetry) {
327+
const statuses = lastStatusByPickleId.get(this.pickle.id)
328+
if (statuses.length === testManagementAttemptToFixRetries + 1) {
329+
// The first status is the original test status
330+
const statusesExceptFirst = statuses.slice(1)
331+
const { pass, fail } = statusesExceptFirst.reduce((acc, status) => {
332+
acc[status]++
333+
return acc
334+
}, { pass: 0, fail: 0 })
335+
hasFailedAllRetries = fail === testManagementAttemptToFixRetries
336+
hasPassedAllRetries = pass === testManagementAttemptToFixRetries
337+
}
320338
}
321339
}
340+
341+
if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) {
342+
const numRetries = numRetriesByPickleId.get(this.pickle.id)
343+
344+
isNew = numRetries !== undefined
345+
isEfdRetry = numRetries > 0
346+
}
322347
const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt)
323348

324349
const error = getErrorFromCucumberResult(result)
@@ -334,6 +359,9 @@ function wrapRun (pl, isLatestVersion) {
334359
isNew,
335360
isEfdRetry,
336361
isFlakyRetry: numAttempt > 0,
362+
isAttemptToFixRetry,
363+
hasFailedAllRetries,
364+
hasPassedAllRetries,
337365
isDisabled,
338366
isQuarantined
339367
})
@@ -426,6 +454,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin
426454
numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount
427455
isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled
428456
isTestManagementTestsEnabled = configurationResponse.libraryConfig?.isTestManagementEnabled
457+
testManagementAttemptToFixRetries = configurationResponse.libraryConfig?.testManagementAttemptToFixRetries
429458

430459
if (isKnownTestsEnabled) {
431460
const knownTestsResponse = await getChannelPromise(knownTestsCh)
@@ -576,29 +605,42 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
576605
}
577606

578607
let isNew = false
608+
let isAttemptToFix = false
579609
let isDisabled = false
580610
let isQuarantined = false
581611

582-
if (isKnownTestsEnabled) {
612+
if (isTestManagementTestsEnabled) {
613+
const testProperties = getTestProperties(testSuitePath, pickle.name)
614+
isAttemptToFix = testProperties.attemptToFix
615+
if (!isAttemptToFix) {
616+
isDisabled = testProperties.disabled
617+
if (isDisabled) {
618+
this.options.dryRun = true
619+
} else {
620+
isQuarantined = testProperties.quarantined
621+
}
622+
}
623+
}
624+
625+
if (isKnownTestsEnabled && !isAttemptToFix) {
583626
isNew = isNewTest(testSuitePath, pickle.name)
584627
if (isNew) {
585628
numRetriesByPickleId.set(pickle.id, 0)
586629
}
587630
}
588-
if (isTestManagementTestsEnabled) {
589-
const testProperties = getTestProperties(testSuitePath, pickle.name)
590-
isDisabled = testProperties.disabled
591-
if (isDisabled) {
592-
this.options.dryRun = true
593-
} else {
594-
isQuarantined = testProperties.quarantined
595-
}
596-
}
597631
// TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId`
598632
let runTestCaseResult = await runTestCaseFunction.apply(this, arguments)
599633

600634
const testStatuses = lastStatusByPickleId.get(pickle.id)
601635
const lastTestStatus = testStatuses[testStatuses.length - 1]
636+
637+
if (isAttemptToFix && lastTestStatus !== 'skip') {
638+
for (let retryIndex = 0; retryIndex < testManagementAttemptToFixRetries; retryIndex++) {
639+
numRetriesByPickleId.set(pickle.id, retryIndex + 1)
640+
runTestCaseResult = await runTestCaseFunction.apply(this, arguments)
641+
}
642+
}
643+
602644
// If it's a new test and it hasn't been skipped, we run it again
603645
if (isEarlyFlakeDetectionEnabled && lastTestStatus !== 'skip' && isNew) {
604646
for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) {
@@ -609,6 +651,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
609651
let testStatus = lastTestStatus
610652
let shouldBePassedByEFD = false
611653
let shouldBePassedByQuarantine = false
654+
let shouldBePassedByAttemptToFix = false
612655
if (isNew && isEarlyFlakeDetectionEnabled) {
613656
/**
614657
* If Early Flake Detection (EFD) is enabled the logic is as follows:
@@ -625,9 +668,17 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
625668
}
626669
}
627670

628-
if (isTestManagementTestsEnabled && isQuarantined) {
629-
this.success = true
630-
shouldBePassedByQuarantine = true
671+
if (isTestManagementTestsEnabled) {
672+
if (isAttemptToFix) {
673+
const testProperties = getTestProperties(testSuitePath, pickle.name)
674+
if (testProperties.disabled || testProperties.quarantined) {
675+
this.success = true
676+
shouldBePassedByAttemptToFix = true
677+
}
678+
} else if (isQuarantined) {
679+
this.success = true
680+
shouldBePassedByQuarantine = true
681+
}
631682
}
632683

633684
if (!pickleResultByFile[testFileAbsolutePath]) {
@@ -661,8 +712,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
661712
return shouldBePassedByEFD
662713
}
663714

664-
if (isNewerCucumberVersion && isTestManagementTestsEnabled && isQuarantined) {
665-
return shouldBePassedByQuarantine
715+
if (isNewerCucumberVersion && isTestManagementTestsEnabled && (isQuarantined || isAttemptToFix)) {
716+
return shouldBePassedByQuarantine || shouldBePassedByAttemptToFix
666717
}
667718

668719
return runTestCaseResult

‎packages/datadog-plugin-cucumber/src/index.js

+20-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ const {
3030
TEST_RETRY_REASON,
3131
TEST_MANAGEMENT_ENABLED,
3232
TEST_MANAGEMENT_IS_QUARANTINED,
33-
TEST_MANAGEMENT_IS_DISABLED
33+
TEST_MANAGEMENT_IS_DISABLED,
34+
TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
35+
TEST_HAS_FAILED_ALL_RETRIES,
36+
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED
3437
} = require('../../dd-trace/src/plugins/util/test')
3538
const { RESOURCE_NAME } = require('../../../ext/tags')
3639
const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants')
@@ -328,6 +331,9 @@ class CucumberPlugin extends CiPlugin {
328331
isNew,
329332
isEfdRetry,
330333
isFlakyRetry,
334+
isAttemptToFixRetry,
335+
hasFailedAllRetries,
336+
hasPassedAllRetries,
331337
isDisabled,
332338
isQuarantined
333339
}) => {
@@ -358,6 +364,19 @@ class CucumberPlugin extends CiPlugin {
358364
span.setTag(TEST_IS_RETRY, 'true')
359365
}
360366

367+
if (hasFailedAllRetries) {
368+
span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true')
369+
}
370+
371+
if (isAttemptToFixRetry) {
372+
span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
373+
span.setTag(TEST_IS_RETRY, 'true')
374+
span.setTag(TEST_RETRY_REASON, 'attempt_to_fix')
375+
if (hasPassedAllRetries) {
376+
span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true')
377+
}
378+
}
379+
361380
if (isDisabled) {
362381
span.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true')
363382
}

0 commit comments

Comments
 (0)