Skip to content

Commit c13d368

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] [SDTEST-1272] Detect new tests regardless of Early Flake Detection (#5138)
1 parent 504b460 commit c13d368

File tree

25 files changed

+1148
-312
lines changed

25 files changed

+1148
-312
lines changed

‎integration-tests/cucumber/cucumber.spec.js

+132-43
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ const {
4242
DI_DEBUG_ERROR_PREFIX,
4343
DI_DEBUG_ERROR_FILE_SUFFIX,
4444
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
45-
DI_DEBUG_ERROR_LINE_SUFFIX
45+
DI_DEBUG_ERROR_LINE_SUFFIX,
46+
TEST_RETRY_REASON
4647
} = require('../../packages/dd-trace/src/plugins/util/test')
4748
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
4849

@@ -844,15 +845,13 @@ versions.forEach(version => {
844845
it('retries new tests', (done) => {
845846
const NUM_RETRIES_EFD = 3
846847
receiver.setSettings({
847-
itr_enabled: false,
848-
code_coverage: false,
849-
tests_skipping: false,
850848
early_flake_detection: {
851849
enabled: true,
852850
slow_test_retries: {
853851
'5s': NUM_RETRIES_EFD
854852
}
855-
}
853+
},
854+
known_tests_enabled: true
856855
})
857856
// cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
858857
receiver.setKnownTests(
@@ -884,6 +883,9 @@ versions.forEach(version => {
884883
retriedTests.length
885884
)
886885
assert.equal(retriedTests.length, NUM_RETRIES_EFD)
886+
retriedTests.forEach(test => {
887+
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd')
888+
})
887889
// Test name does not change
888890
newTests.forEach(test => {
889891
assert.equal(test.meta[TEST_NAME], 'Say whatever')
@@ -907,15 +909,13 @@ versions.forEach(version => {
907909
it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => {
908910
const NUM_RETRIES_EFD = 3
909911
receiver.setSettings({
910-
itr_enabled: false,
911-
code_coverage: false,
912-
tests_skipping: false,
913912
early_flake_detection: {
914913
enabled: true,
915914
slow_test_retries: {
916915
'5s': NUM_RETRIES_EFD
917916
}
918-
}
917+
},
918+
known_tests_enabled: true
919919
})
920920

921921
const eventsPromise = receiver
@@ -928,8 +928,12 @@ versions.forEach(version => {
928928
const newTests = tests.filter(test =>
929929
test.meta[TEST_IS_NEW] === 'true'
930930
)
931-
// new tests are not detected
932-
assert.equal(newTests.length, 0)
931+
// new tests are detected but not retried
932+
assert.equal(newTests.length, 1)
933+
const retriedTests = tests.filter(test =>
934+
test.meta[TEST_IS_RETRY] === 'true'
935+
)
936+
assert.equal(retriedTests.length, 0)
933937
})
934938
// cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
935939
receiver.setKnownTests({
@@ -957,15 +961,13 @@ versions.forEach(version => {
957961
it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => {
958962
const NUM_RETRIES_EFD = 3
959963
receiver.setSettings({
960-
itr_enabled: false,
961-
code_coverage: false,
962-
tests_skipping: false,
963964
early_flake_detection: {
964965
enabled: true,
965966
slow_test_retries: {
966967
'5s': NUM_RETRIES_EFD
967968
}
968-
}
969+
},
970+
known_tests_enabled: true
969971
})
970972
// Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new
971973
receiver.setKnownTests({})
@@ -1014,15 +1016,13 @@ versions.forEach(version => {
10141016
it('does not retry tests that are skipped', (done) => {
10151017
const NUM_RETRIES_EFD = 3
10161018
receiver.setSettings({
1017-
itr_enabled: false,
1018-
code_coverage: false,
1019-
tests_skipping: false,
10201019
early_flake_detection: {
10211020
enabled: true,
10221021
slow_test_retries: {
10231022
'5s': NUM_RETRIES_EFD
10241023
}
1025-
}
1024+
},
1025+
known_tests_enabled: true
10261026
})
10271027
// "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new
10281028
// "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new
@@ -1066,15 +1066,13 @@ versions.forEach(version => {
10661066
it('does not run EFD if the known tests request fails', (done) => {
10671067
const NUM_RETRIES_EFD = 3
10681068
receiver.setSettings({
1069-
itr_enabled: false,
1070-
code_coverage: false,
1071-
tests_skipping: false,
10721069
early_flake_detection: {
10731070
enabled: true,
10741071
slow_test_retries: {
10751072
'5s': NUM_RETRIES_EFD
10761073
}
1077-
}
1074+
},
1075+
known_tests_enabled: true
10781076
})
10791077
receiver.setKnownTestsResponseCode(500)
10801078
receiver.setKnownTests({})
@@ -1108,16 +1106,14 @@ versions.forEach(version => {
11081106
it('bails out of EFD if the percentage of new tests is too high', (done) => {
11091107
const NUM_RETRIES_EFD = 3
11101108
receiver.setSettings({
1111-
itr_enabled: false,
1112-
code_coverage: false,
1113-
tests_skipping: false,
11141109
early_flake_detection: {
11151110
enabled: true,
11161111
slow_test_retries: {
11171112
'5s': NUM_RETRIES_EFD
11181113
},
11191114
faulty_session_threshold: 0
1120-
}
1115+
},
1116+
known_tests_enabled: true
11211117
})
11221118
// tests in cucumber.ci-visibility/features/farewell.feature will be considered new
11231119
receiver.setKnownTests(
@@ -1160,20 +1156,70 @@ versions.forEach(version => {
11601156
})
11611157
})
11621158

1159+
it('disables early flake detection if known tests should not be requested', (done) => {
1160+
const NUM_RETRIES_EFD = 3
1161+
receiver.setSettings({
1162+
early_flake_detection: {
1163+
enabled: true,
1164+
slow_test_retries: {
1165+
'5s': NUM_RETRIES_EFD
1166+
}
1167+
},
1168+
known_tests_enabled: false
1169+
})
1170+
// cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
1171+
receiver.setKnownTests(
1172+
{
1173+
cucumber: {
1174+
'ci-visibility/features/farewell.feature': ['Say farewell'],
1175+
'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip']
1176+
}
1177+
}
1178+
)
1179+
const eventsPromise = receiver
1180+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
1181+
const events = payloads.flatMap(({ payload }) => payload.events)
1182+
1183+
const testSession = events.find(event => event.type === 'test_session_end').content
1184+
assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
1185+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1186+
1187+
// no new tests detected
1188+
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
1189+
assert.equal(newTests.length, 0)
1190+
// no retries
1191+
const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1192+
assert.equal(retriedTests.length, 0)
1193+
})
1194+
1195+
childProcess = exec(
1196+
runTestsCommand,
1197+
{
1198+
cwd,
1199+
env: envVars,
1200+
stdio: 'pipe'
1201+
}
1202+
)
1203+
1204+
childProcess.on('exit', () => {
1205+
eventsPromise.then(() => {
1206+
done()
1207+
}).catch(done)
1208+
})
1209+
})
1210+
11631211
if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11
11641212
context('parallel mode', () => {
11651213
it('retries new tests', (done) => {
11661214
const NUM_RETRIES_EFD = 3
11671215
receiver.setSettings({
1168-
itr_enabled: false,
1169-
code_coverage: false,
1170-
tests_skipping: false,
11711216
early_flake_detection: {
11721217
enabled: true,
11731218
slow_test_retries: {
11741219
'5s': NUM_RETRIES_EFD
11751220
}
1176-
}
1221+
},
1222+
known_tests_enabled: true
11771223
})
11781224
// cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
11791225
receiver.setKnownTests(
@@ -1231,15 +1277,13 @@ versions.forEach(version => {
12311277
it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => {
12321278
const NUM_RETRIES_EFD = 3
12331279
receiver.setSettings({
1234-
itr_enabled: false,
1235-
code_coverage: false,
1236-
tests_skipping: false,
12371280
early_flake_detection: {
12381281
enabled: true,
12391282
slow_test_retries: {
12401283
'5s': NUM_RETRIES_EFD
12411284
}
1242-
}
1285+
},
1286+
known_tests_enabled: true
12431287
})
12441288
// Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new
12451289
receiver.setKnownTests({})
@@ -1293,16 +1337,14 @@ versions.forEach(version => {
12931337
it('bails out of EFD if the percentage of new tests is too high', (done) => {
12941338
const NUM_RETRIES_EFD = 3
12951339
receiver.setSettings({
1296-
itr_enabled: false,
1297-
code_coverage: false,
1298-
tests_skipping: false,
12991340
early_flake_detection: {
13001341
enabled: true,
13011342
slow_test_retries: {
13021343
'5s': NUM_RETRIES_EFD
13031344
},
13041345
faulty_session_threshold: 0
1305-
}
1346+
},
1347+
known_tests_enabled: true
13061348
})
13071349
// tests in cucumber.ci-visibility/features/farewell.feature will be considered new
13081350
receiver.setKnownTests(
@@ -1350,15 +1392,13 @@ versions.forEach(version => {
13501392
it('does not retry tests that are skipped', (done) => {
13511393
const NUM_RETRIES_EFD = 3
13521394
receiver.setSettings({
1353-
itr_enabled: false,
1354-
code_coverage: false,
1355-
tests_skipping: false,
13561395
early_flake_detection: {
13571396
enabled: true,
13581397
slow_test_retries: {
13591398
'5s': NUM_RETRIES_EFD
13601399
}
1361-
}
1400+
},
1401+
known_tests_enabled: true
13621402
})
13631403
// "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new
13641404
// "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new
@@ -1909,5 +1949,54 @@ versions.forEach(version => {
19091949
})
19101950
})
19111951
})
1952+
1953+
context('known tests without early flake detection', () => {
1954+
it('detects new tests without retrying them', (done) => {
1955+
receiver.setSettings({
1956+
early_flake_detection: {
1957+
enabled: false
1958+
},
1959+
known_tests_enabled: true
1960+
})
1961+
// cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
1962+
receiver.setKnownTests(
1963+
{
1964+
cucumber: {
1965+
'ci-visibility/features/farewell.feature': ['Say farewell'],
1966+
'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip']
1967+
}
1968+
}
1969+
)
1970+
const eventsPromise = receiver
1971+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
1972+
const events = payloads.flatMap(({ payload }) => payload.events)
1973+
1974+
const testSession = events.find(event => event.type === 'test_session_end').content
1975+
assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
1976+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1977+
1978+
// new tests detected but not retried
1979+
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
1980+
assert.equal(newTests.length, 1)
1981+
const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1982+
assert.equal(retriedTests.length, 0)
1983+
})
1984+
1985+
childProcess = exec(
1986+
runTestsCommand,
1987+
{
1988+
cwd,
1989+
env: getCiVisAgentlessConfig(receiver.port),
1990+
stdio: 'pipe'
1991+
}
1992+
)
1993+
1994+
childProcess.on('exit', () => {
1995+
eventsPromise.then(() => {
1996+
done()
1997+
}).catch(done)
1998+
})
1999+
})
2000+
})
19122001
})
19132002
})

0 commit comments

Comments
 (0)