Skip to content

Commit e6ea5fd

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Change failed test replay to opt out rather than opt in (#5283)
1 parent 14a3795 commit e6ea5fd

File tree

8 files changed

+36
-49
lines changed

8 files changed

+36
-49
lines changed

‎integration-tests/cucumber/cucumber.spec.js

+8-14
Original file line numberDiff line numberDiff line change
@@ -1587,7 +1587,7 @@ versions.forEach(version => {
15871587
})
15881588
// Dynamic instrumentation only supported from >=8.0.0
15891589
context('dynamic instrumentation', () => {
1590-
it('does not activate if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
1590+
it('does not activate if DD_TEST_FAILED_TEST_REPLAY_ENABLED is set to false', (done) => {
15911591
receiver.setSettings({
15921592
flaky_test_retries_enabled: true,
15931593
di_enabled: true
@@ -1621,7 +1621,10 @@ versions.forEach(version => {
16211621
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
16221622
{
16231623
cwd,
1624-
env: envVars,
1624+
env: {
1625+
...envVars,
1626+
DD_TEST_FAILED_TEST_REPLAY_ENABLED: 'false'
1627+
},
16251628
stdio: 'pipe'
16261629
}
16271630
)
@@ -1666,10 +1669,7 @@ versions.forEach(version => {
16661669
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
16671670
{
16681671
cwd,
1669-
env: {
1670-
...envVars,
1671-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
1672-
},
1672+
env: envVars,
16731673
stdio: 'pipe'
16741674
}
16751675
)
@@ -1748,10 +1748,7 @@ versions.forEach(version => {
17481748
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
17491749
{
17501750
cwd,
1751-
env: {
1752-
...envVars,
1753-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
1754-
},
1751+
env: envVars,
17551752
stdio: 'pipe'
17561753
}
17571754
)
@@ -1800,10 +1797,7 @@ versions.forEach(version => {
18001797
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-not-hit-breakpoint.feature --retry 1',
18011798
{
18021799
cwd,
1803-
env: {
1804-
...envVars,
1805-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
1806-
},
1800+
env: envVars,
18071801
stdio: 'pipe'
18081802
}
18091803
)

‎integration-tests/jest/jest.spec.js

+4-8
Original file line numberDiff line numberDiff line change
@@ -509,7 +509,7 @@ describe('jest CommonJS', () => {
509509
}).catch(done)
510510
})
511511

512-
it('can work with Dynamic Instrumentation', (done) => {
512+
it('can work with Failed Test Replay', (done) => {
513513
receiver.setSettings({
514514
flaky_test_retries_enabled: true,
515515
di_enabled: true
@@ -565,7 +565,6 @@ describe('jest CommonJS', () => {
565565
env: {
566566
...getCiVisAgentlessConfig(receiver.port),
567567
TESTS_TO_RUN: 'dynamic-instrumentation/test-',
568-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
569568
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1',
570569
RUN_IN_PARALLEL: true
571570
},
@@ -2537,7 +2536,7 @@ describe('jest CommonJS', () => {
25372536
})
25382537

25392538
context('dynamic instrumentation', () => {
2540-
it('does not activate dynamic instrumentation if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
2539+
it('does not activate dynamic instrumentation if DD_TEST_FAILED_TEST_REPLAY_ENABLED is set to false', (done) => {
25412540
receiver.setSettings({
25422541
flaky_test_retries_enabled: true,
25432542
di_enabled: true
@@ -2571,7 +2570,8 @@ describe('jest CommonJS', () => {
25712570
env: {
25722571
...getCiVisAgentlessConfig(receiver.port),
25732572
TESTS_TO_RUN: 'dynamic-instrumentation/test-hit-breakpoint',
2574-
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
2573+
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1',
2574+
DD_TEST_FAILED_TEST_REPLAY_ENABLED: 'false'
25752575
},
25762576
stdio: 'inherit'
25772577
}
@@ -2618,7 +2618,6 @@ describe('jest CommonJS', () => {
26182618
env: {
26192619
...getCiVisAgentlessConfig(receiver.port),
26202620
TESTS_TO_RUN: 'dynamic-instrumentation/test-hit-breakpoint',
2621-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
26222621
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
26232622
},
26242623
stdio: 'inherit'
@@ -2703,7 +2702,6 @@ describe('jest CommonJS', () => {
27032702
env: {
27042703
...getCiVisAgentlessConfig(receiver.port),
27052704
TESTS_TO_RUN: 'dynamic-instrumentation/test-hit-breakpoint',
2706-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
27072705
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
27082706
},
27092707
stdio: 'inherit'
@@ -2753,7 +2751,6 @@ describe('jest CommonJS', () => {
27532751
env: {
27542752
...getCiVisAgentlessConfig(receiver.port),
27552753
TESTS_TO_RUN: 'dynamic-instrumentation/test-not-hit-breakpoint',
2756-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
27572754
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
27582755
},
27592756
stdio: 'inherit'
@@ -2793,7 +2790,6 @@ describe('jest CommonJS', () => {
27932790
env: {
27942791
...getCiVisAgentlessConfig(receiver.port),
27952792
TESTS_TO_RUN: 'dynamic-instrumentation/test-hit-breakpoint',
2796-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
27972793
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1',
27982794
TEST_SHOULD_PASS_AFTER_RETRY: '1'
27992795
},

‎integration-tests/mocha/mocha.spec.js

+3-5
Original file line numberDiff line numberDiff line change
@@ -2209,7 +2209,7 @@ describe('mocha CommonJS', function () {
22092209
})
22102210

22112211
context('dynamic instrumentation', () => {
2212-
it('does not activate dynamic instrumentation if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
2212+
it('does not activate dynamic instrumentation if DD_TEST_FAILED_TEST_REPLAY_ENABLED is set to false', (done) => {
22132213
receiver.setSettings({
22142214
flaky_test_retries_enabled: true,
22152215
di_enabled: true
@@ -2247,7 +2247,8 @@ describe('mocha CommonJS', function () {
22472247
TESTS_TO_RUN: JSON.stringify([
22482248
'./dynamic-instrumentation/test-hit-breakpoint'
22492249
]),
2250-
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
2250+
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1',
2251+
DD_TEST_FAILED_TEST_REPLAY_ENABLED: 'false'
22512252
},
22522253
stdio: 'inherit'
22532254
}
@@ -2299,7 +2300,6 @@ describe('mocha CommonJS', function () {
22992300
TESTS_TO_RUN: JSON.stringify([
23002301
'./dynamic-instrumentation/test-hit-breakpoint'
23012302
]),
2302-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
23032303
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
23042304
},
23052305
stdio: 'inherit'
@@ -2389,7 +2389,6 @@ describe('mocha CommonJS', function () {
23892389
TESTS_TO_RUN: JSON.stringify([
23902390
'./dynamic-instrumentation/test-hit-breakpoint'
23912391
]),
2392-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
23932392
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
23942393
},
23952394
stdio: 'inherit'
@@ -2443,7 +2442,6 @@ describe('mocha CommonJS', function () {
24432442
TESTS_TO_RUN: JSON.stringify([
24442443
'./dynamic-instrumentation/test-not-hit-breakpoint'
24452444
]),
2446-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true',
24472445
DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '1'
24482446
},
24492447
stdio: 'inherit'

‎integration-tests/vitest/vitest.spec.js

+6-8
Original file line numberDiff line numberDiff line change
@@ -987,7 +987,7 @@ versions.forEach((version) => {
987987
// dynamic instrumentation only supported from >=2.0.0
988988
if (version === 'latest') {
989989
context('dynamic instrumentation', () => {
990-
it('does not activate it if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
990+
it('does not activate it if DD_TEST_FAILED_TEST_REPLAY_ENABLED is set to false', (done) => {
991991
receiver.setSettings({
992992
flaky_test_retries_enabled: true,
993993
di_enabled: true
@@ -1025,7 +1025,8 @@ versions.forEach((version) => {
10251025
env: {
10261026
...getCiVisAgentlessConfig(receiver.port),
10271027
TEST_DIR: 'ci-visibility/vitest-tests/dynamic-instrumentation*',
1028-
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init'
1028+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
1029+
DD_TEST_FAILED_TEST_REPLAY_ENABLED: 'false'
10291030
},
10301031
stdio: 'pipe'
10311032
}
@@ -1075,8 +1076,7 @@ versions.forEach((version) => {
10751076
env: {
10761077
...getCiVisAgentlessConfig(receiver.port),
10771078
TEST_DIR: 'ci-visibility/vitest-tests/dynamic-instrumentation*',
1078-
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
1079-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: '1'
1079+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init'
10801080
},
10811081
stdio: 'pipe'
10821082
}
@@ -1162,8 +1162,7 @@ versions.forEach((version) => {
11621162
env: {
11631163
...getCiVisAgentlessConfig(receiver.port),
11641164
TEST_DIR: 'ci-visibility/vitest-tests/dynamic-instrumentation*',
1165-
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
1166-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: '1'
1165+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init'
11671166
},
11681167
stdio: 'pipe'
11691168
}
@@ -1217,8 +1216,7 @@ versions.forEach((version) => {
12171216
env: {
12181217
...getCiVisAgentlessConfig(receiver.port),
12191218
TEST_DIR: 'ci-visibility/vitest-tests/breakpoint-not-hit*',
1220-
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
1221-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: '1'
1219+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init'
12221220
},
12231221
stdio: 'pipe'
12241222
}

‎packages/dd-trace/src/ci-visibility/dynamic-instrumentation/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ class TestVisDynamicInstrumentation {
7070
env: {
7171
...process.env,
7272
DD_TRACE_ENABLED: 0,
73-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 0
73+
DD_TEST_FAILED_TEST_REPLAY_ENABLED: 0
7474
},
7575
workerData: {
7676
config: config.serialize(),

‎packages/dd-trace/src/config.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1144,7 +1144,7 @@ class Config {
11441144
DD_CIVISIBILITY_FLAKY_RETRY_COUNT,
11451145
DD_TEST_SESSION_NAME,
11461146
DD_AGENTLESS_LOG_SUBMISSION_ENABLED,
1147-
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED,
1147+
DD_TEST_FAILED_TEST_REPLAY_ENABLED,
11481148
DD_TEST_MANAGEMENT_ENABLED,
11491149
DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES
11501150
} = process.env
@@ -1164,7 +1164,7 @@ class Config {
11641164
this._setBoolean(calc, 'isManualApiEnabled', !isFalse(this._isCiVisibilityManualApiEnabled()))
11651165
this._setString(calc, 'ciVisibilityTestSessionName', DD_TEST_SESSION_NAME)
11661166
this._setBoolean(calc, 'ciVisAgentlessLogSubmissionEnabled', isTrue(DD_AGENTLESS_LOG_SUBMISSION_ENABLED))
1167-
this._setBoolean(calc, 'isTestDynamicInstrumentationEnabled', isTrue(DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED))
1167+
this._setBoolean(calc, 'isTestDynamicInstrumentationEnabled', !isFalse(DD_TEST_FAILED_TEST_REPLAY_ENABLED))
11681168
this._setBoolean(calc, 'isServiceUserProvided', !!this._env.service)
11691169
this._setBoolean(calc, 'isTestManagementEnabled', !isFalse(DD_TEST_MANAGEMENT_ENABLED))
11701170
this._setValue(calc,

‎packages/dd-trace/src/plugins/ci_plugin.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -202,15 +202,15 @@ module.exports = class CiPlugin extends Plugin {
202202
configure (config, shouldGetEnvironmentData = true) {
203203
super.configure(config)
204204

205+
if (!shouldGetEnvironmentData) {
206+
return
207+
}
208+
205209
if (config.isTestDynamicInstrumentationEnabled && !this.di) {
206210
const testVisibilityDynamicInstrumentation = require('../ci-visibility/dynamic-instrumentation')
207211
this.di = testVisibilityDynamicInstrumentation
208212
}
209213

210-
if (!shouldGetEnvironmentData) {
211-
return
212-
}
213-
214214
this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config)
215215

216216
const {

‎packages/dd-trace/test/config.spec.js

+8-7
Original file line numberDiff line numberDiff line change
@@ -2039,7 +2039,7 @@ describe('Config', () => {
20392039
delete process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT
20402040
delete process.env.DD_TEST_SESSION_NAME
20412041
delete process.env.JEST_WORKER_ID
2042-
delete process.env.DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED
2042+
delete process.env.DD_TEST_FAILED_TEST_REPLAY_ENABLED
20432043
delete process.env.DD_AGENTLESS_LOG_SUBMISSION_ENABLED
20442044
options = {}
20452045
})
@@ -2138,15 +2138,16 @@ describe('Config', () => {
21382138
const config = new Config(options)
21392139
expect(config).to.have.property('ciVisAgentlessLogSubmissionEnabled', true)
21402140
})
2141-
it('should not set isTestDynamicInstrumentationEnabled by default', () => {
2142-
const config = new Config(options)
2143-
expect(config).to.have.property('isTestDynamicInstrumentationEnabled', false)
2144-
})
2145-
it('should set isTestDynamicInstrumentationEnabled if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is passed', () => {
2146-
process.env.DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED = 'true'
2141+
it('should set isTestDynamicInstrumentationEnabled by default', () => {
21472142
const config = new Config(options)
21482143
expect(config).to.have.property('isTestDynamicInstrumentationEnabled', true)
21492144
})
2145+
it('should set isTestDynamicInstrumentationEnabled to false if DD_TEST_FAILED_TEST_REPLAY_ENABLED is false',
2146+
() => {
2147+
process.env.DD_TEST_FAILED_TEST_REPLAY_ENABLED = 'false'
2148+
const config = new Config(options)
2149+
expect(config).to.have.property('isTestDynamicInstrumentationEnabled', false)
2150+
})
21502151
})
21512152
context('ci visibility mode is not enabled', () => {
21522153
it('should not activate intelligent test runner or git metadata upload', () => {

0 commit comments

Comments
 (0)