[Test Optimization] - Code refactor (#6709)

## Summary of changes

https://datadoghq.atlassian.net/browse/SDTEST-1423

This PR introduces a complete refactor of the existing two classes:
- CIVisibility.cs
- IntelligentTestRunnerClient.cs

Alsos rename the following classes:
- TestOptimizationTracerManager.cs
- TestOptimizationTracerManagerFactory.cs
- TestOptimizationSettings.cs

With the refactor the following changes were made:

#### CIVisibility.cs
- Removed the static CIVisibility class.
- Added the following abstractions:
- [x] ITestOptimization.cs
- [x] ITestOptimizationHostInfo.cs
- [x] ITestOptimizationTracerManagement.cs
- [x] ITestOptimizationFeature.cs
- [x] ITestOptimizationEarlyFlakeDetectionFeature.cs
- [x] ITestOptimizationFlakyRetryFeature.cs
- [x] ITestOptimizationImpactedTestsDetectionFeature.cs
- [x] ITestOptimizationSkippableFeature.cs
- Also added an implementation to the abstractions, and removed the
static approach with a singleton, similar to the tracer class.

#### IntelligentTestRunnerClient.cs
- Removed the IntelligentTestRunnerClient.cs class
- Added the following files:
- [x] ITestOptimizationClient.cs : an abstraction for the http client
for all test optimization transactions.
- [x] NoopTestOptimizationClient.cs: a NoOp implementation of the
client.
- [x] CachedTestOptimizationClient.cs: a cache layer over an
ITestOptimizationClient.cs
- [x] TestOptimizationClient.cs: the default implementation partial
class, this part handles the basic http operations with retries code.
- [x] TestOptimizationClient.GetCommitsAsync.cs: the default
implementation partial class, this part handles the GetCommitsAsync
transaction.
- [x] TestOptimizationClient.GetEarlyFlakeDetectionTestsAsync.cs: the
default implementation partial class, this part handles the
GetEarlyFlakeDetectionTestsAsync transaction.
- [x] TestOptimizationClient.GetImpactedTestsDetectionFilesAsync.cs: the
default implementation partial class, this part handles the
GetImpactedTestsDetectionFilesAsync transaction.
- [x] TestOptimizationClient.GetSettingsAsync.cs: the default
implementation partial class, this part handles the GetSettingsAsync
transaction.
- [x] TestOptimizationClient.GetSkippableTestsAsync.cs: the default
implementation partial class, this part handles the
GetSkippableTestsAsync transaction.
- [x] TestOptimizationClient.SendPackFilesAsync.cs: the default
implementation partial class, this part handles the SendPackFilesAsync
transaction.
- [x] TestOptimizationClient.UploadRepositoryChangesAsync.cs: the
default implementation partial class, this part handles the
UploadRepositoryChangesAsync transaction.

## Reason for change

The current code is already complex and hard to maintain, all features
are included in only two classes. Before introducing more features in
the product (KnownTests, FlakyTestManagement) we need to improve the
existing code and make our life easier.

## Implementation details

It's the same code sparse in different files with a better structure for
both adding new features and testing.

## Test coverage

Same tests must behave the same we are not looking for a change of the
current behavior.

## Other details
<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

Author: tonyredondo

tracer/build/_build/Build.Steps.cs

@@ -2249,12 +2249,13 @@ string NormalizedPath(AbsolutePath ap)
                new(@".*System.Threading.ThreadAbortException: Thread was being aborted\.", RegexOptions.Compiled),
                new(@".*System.InvalidOperationException: Module Samples.Trimming.dll has no HINSTANCE.*", RegexOptions.Compiled),
                // CI Visibility known errors
-               new (@".*The Git repository couldn't be automatically extracted.*", RegexOptions.Compiled),
-               new (@".*DD_GIT_REPOSITORY_URL is set with.*", RegexOptions.Compiled),
-               new (@".*The Git commit sha couldn't be automatically extracted.*", RegexOptions.Compiled),
-               new (@".*DD_GIT_COMMIT_SHA must be a full-length git SHA.*", RegexOptions.Compiled),
-               new (@".*Timeout occurred when flushing spans.*", RegexOptions.Compiled),
-               new (@".*ITR: .*", RegexOptions.Compiled),
+               new(@".*The Git repository couldn't be automatically extracted.*", RegexOptions.Compiled),
+               new(@".*DD_GIT_REPOSITORY_URL is set with.*", RegexOptions.Compiled),
+               new(@".*The Git commit sha couldn't be automatically extracted.*", RegexOptions.Compiled),
+               new(@".*DD_GIT_COMMIT_SHA must be a full-length git SHA.*", RegexOptions.Compiled),
+               new(@".*Timeout occurred when flushing spans.*", RegexOptions.Compiled),
+               new(@".*TestOptimization: .*", RegexOptions.Compiled),
+               new(@".*TestOptimizationClient: .*", RegexOptions.Compiled),
                // This one is annoying but we _think_ due to a dodgy named pipes implementation, so ignoring for now
                new(@".*An error occurred while sending data to the agent at \\\\\.\\pipe\\trace-.*The operation has timed out.*", RegexOptions.Compiled),
                new(@".*An error occurred while sending data to the agent at \\\\\.\\pipe\\metrics-.*The operation has timed out.*", RegexOptions.Compiled),
@@ -2312,7 +2313,8 @@ string NormalizedPath(AbsolutePath ap)
            knownPatterns.Add(new(@".*The Git commit sha couldn't be automatically extracted.*", RegexOptions.Compiled));
            knownPatterns.Add(new(@".*DD_GIT_COMMIT_SHA must be a full-length git SHA.*", RegexOptions.Compiled));
            knownPatterns.Add(new(@".*Timeout occurred when flushing spans.*", RegexOptions.Compiled));
-           knownPatterns.Add(new(@".*ITR: .*", RegexOptions.Compiled));
+           knownPatterns.Add(new(@".*TestOptimization: .*", RegexOptions.Compiled));
+           knownPatterns.Add(new(@".*TestOptimizationClient: .*", RegexOptions.Compiled));
 
            CheckLogsForErrors(knownPatterns, allFilesMustExist: true, minLogLevel: LogLevel.Warning);
        });

tracer/src/Datadog.Trace.BenchmarkDotNet/BenchmarkMetadata.cs

@@ -2,6 +2,7 @@
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
+
 #nullable enable
 
 using System;
@@ -18,15 +19,15 @@ internal static class BenchmarkMetadata
     static BenchmarkMetadata()
     {
         MetadataByBenchmark = new();
-        CIVisibility.InitializeFromManualInstrumentation();
+        TestOptimization.Instance.InitializeFromManualInstrumentation();
     }
 
     public static void GetIds(object key, out TraceId traceId, out ulong spanId)
     {
         var value = MetadataByBenchmark.GetOrAdd(key, @case => new());
         if (value.TraceId is null)
         {
-            var useAllBits = CIVisibility.Settings.TracerSettings?.TraceId128BitGenerationEnabled ?? true;
+            var useAllBits = TestOptimization.Instance.Settings.TracerSettings?.TraceId128BitGenerationEnabled ?? true;
             value.TraceId = RandomIdGenerator.Shared.NextTraceId(useAllBits);
             value.SpanId = RandomIdGenerator.Shared.NextSpanId(useAllBits);
         }

tracer/src/Datadog.Trace.BenchmarkDotNet/DatadogDiagnoser.cs

@@ -301,7 +301,7 @@ private static void SetEnvironmentVariables(DiagnoserActionParameters parameters
             environment[profilerHeapEnabled] = "1";
         }
 
-        environment["DD_PROFILING_AGENTLESS"] = CIVisibility.Settings.Agentless ? "1" : "0";
+        environment["DD_PROFILING_AGENTLESS"] = TestOptimization.Instance.Settings.Agentless ? "1" : "0";
         environment["DD_PROFILING_UPLOAD_PERIOD"] = "90";
         environment["DD_INTERNAL_PROFILING_SAMPLING_RATE"] = "1";
         environment["DD_INTERNAL_PROFILING_WALLTIME_THREADS_THRESHOLD"] = "64";

tracer/src/Datadog.Trace.Coverage.collector/AssemblyProcessor.cs

@@ -64,9 +64,9 @@ public AssemblyProcessor(string filePath, CoverageSettings settings, ICollectorL
             _settings = settings;
             _logger = logger ?? new ConsoleCollectorLogger();
             _assemblyFilePath = filePath ?? throw new ArgumentNullException(nameof(filePath));
-            _enableJitOptimizations = settings.CIVisibility.CodeCoverageEnableJitOptimizations;
+            _enableJitOptimizations = settings.TestOptimization.CodeCoverageEnableJitOptimizations;
             _coverageMode = CoverageMode.LineExecution;
-            if (settings.CIVisibility.CodeCoverageMode is { Length: > 0 } strCodeCoverageMode)
+            if (settings.TestOptimization.CodeCoverageMode is { Length: > 0 } strCodeCoverageMode)
             {
                 if (Enum.TryParse<CoverageMode>(strCodeCoverageMode, ignoreCase: true, out var coverageMode))
                 {
@@ -169,7 +169,7 @@ public unsafe void Process()
                 {
                     _logger.Debug($"Assembly: {FilePath} is signed.");
 
-                    var snkFilePath = _settings.CIVisibility.CodeCoverageSnkFilePath;
+                    var snkFilePath = _settings.TestOptimization.CodeCoverageSnkFilePath;
                     _logger.Debug($"Assembly: {FilePath} loading .snk file: {snkFilePath}.");
                     if (!string.IsNullOrWhiteSpace(snkFilePath) && File.Exists(snkFilePath))
                     {

tracer/src/Datadog.Trace.Coverage.collector/CoverageSettings.cs

@@ -15,10 +15,10 @@ namespace Datadog.Trace.Coverage.Collector;
 /// </summary>
 internal class CoverageSettings
 {
-    public CoverageSettings(XmlElement? configurationElement, string? tracerHome, CIVisibilitySettings? ciVisibilitySettings = null)
+    public CoverageSettings(XmlElement? configurationElement, string? tracerHome, TestOptimizationSettings? testOptimizationSettings = null)
     {
         TracerHome = tracerHome;
-        CIVisibility = ciVisibilitySettings ?? CIVisibilitySettings.FromDefaultSources();
+        TestOptimization = testOptimizationSettings ?? TestOptimizationSettings.FromDefaultSources();
 
         if (configurationElement is not null)
         {
@@ -43,7 +43,7 @@ public CoverageSettings(XmlElement? configurationElement, string? tracerHome, CI
     }
 
     public CoverageSettings(XmlElement? configurationElement)
-        : this(configurationElement, Util.EnvironmentHelpers.GetEnvironmentVariable("DD_DOTNET_TRACER_HOME"), CIVisibilitySettings.FromDefaultSources())
+        : this(configurationElement, Util.EnvironmentHelpers.GetEnvironmentVariable("DD_DOTNET_TRACER_HOME"), TestOptimizationSettings.FromDefaultSources())
     {
     }
 
@@ -70,7 +70,7 @@ public CoverageSettings(XmlElement? configurationElement)
     /// <summary>
     /// Gets the CI Visibility settings
     /// </summary>
-    public CIVisibilitySettings CIVisibility { get; }
+    public TestOptimizationSettings TestOptimization { get; }
 
     private static void GetStringArrayFromXmlElement(XmlElement? xmlElement, ref IReadOnlyList<string> elements, Func<string?, bool>? validator = null)
     {

tracer/src/Datadog.Trace.MSBuild/DatadogLogger.cs

@@ -47,7 +47,7 @@ static DatadogLogger()
                 // .
             }
 
-            CIVisibility.Initialize();
+            TestOptimization.Instance.Initialize();
         }
 
         /// <summary>

tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs

@@ -2,6 +2,7 @@
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
+
 #nullable enable
 
 using System;
@@ -15,6 +16,7 @@
 using Datadog.Trace.Ci;
 using Datadog.Trace.Ci.CiEnvironment;
 using Datadog.Trace.Ci.Configuration;
+using Datadog.Trace.Ci.Net;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Util;
 using Spectre.Console;
@@ -37,30 +39,33 @@ public static async Task<InitResults> InitializeCiCommandsAsync(
         // Define the arguments
         var lstArguments = new List<string>(args.Length > 1 ? args.Skip(1) : []);
 
-        // CI Visibility mode is enabled.
-        var ciVisibilitySettings = CIVisibility.Settings;
+        // Test optimization instance.
+        var testOptimization = TestOptimization.Instance;
+
+        // Test optimization mode is enabled.
+        var testOptimizationSettings = testOptimization.Settings;
 
         // Get profiler environment variables
         if (!RunHelper.TryGetEnvironmentVariables(
                 applicationContext,
                 context,
                 commonTracerSettings,
-                new Utils.CIVisibilityOptions(ciVisibilitySettings.InstallDatadogTraceInGac, true, reducePathLength),
+                new Utils.CIVisibilityOptions(testOptimizationSettings.InstallDatadogTraceInGac, true, reducePathLength),
                 out var profilerEnvironmentVariables))
         {
             context.ExitCode = 1;
             return new InitResults(false, lstArguments, null, false, false, Task.CompletedTask);
         }
 
-        // Reload the CI Visibility settings (in case they were changed by the environment variables using the `--set-env` option)
-        ciVisibilitySettings = CIVisibilitySettings.FromDefaultSources();
+        // Reload the Test optimization settings (in case they were changed by the environment variables using the `--set-env` option)
+        testOptimizationSettings = TestOptimizationSettings.FromDefaultSources();
 
-        // We force CIVisibility mode on child process
+        // We force Test optimization mode on child process
         profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.Enabled] = "1";
 
         // We check the settings and merge with the command settings options
-        var agentless = ciVisibilitySettings.Agentless;
-        var apiKey = ciVisibilitySettings.ApiKey;
+        var agentless = testOptimizationSettings.Agentless;
+        var apiKey = testOptimizationSettings.ApiKey;
 
         var customApiKey = apiKeyOption?.GetValue(context);
         if (!string.IsNullOrEmpty(customApiKey))
@@ -103,25 +108,25 @@ public static async Task<InitResults> InitializeCiCommandsAsync(
         var uploadRepositoryChangesTask = Task.CompletedTask;
 
         // Set Agentless configuration from the command line options
-        ciVisibilitySettings.SetAgentlessConfiguration(agentless, apiKey, ciVisibilitySettings.AgentlessUrl);
+        testOptimizationSettings.SetAgentlessConfiguration(agentless, apiKey, testOptimizationSettings.AgentlessUrl);
 
         if (!string.IsNullOrEmpty(agentUrl))
         {
             EnvironmentHelpers.SetEnvironmentVariable(Configuration.ConfigurationKeys.AgentUri, agentUrl);
         }
 
         // Initialize flags to enable code coverage and test skipping
-        var codeCoverageEnabled = ciVisibilitySettings.CodeCoverageEnabled == true || ciVisibilitySettings.TestsSkippingEnabled == true;
-        var testSkippingEnabled = ciVisibilitySettings.TestsSkippingEnabled == true;
-        var earlyFlakeDetectionEnabled = ciVisibilitySettings.EarlyFlakeDetectionEnabled == true;
-        var flakyRetryEnabled = ciVisibilitySettings.FlakyRetryEnabled == true;
+        var codeCoverageEnabled = testOptimizationSettings.CodeCoverageEnabled == true || testOptimizationSettings.TestsSkippingEnabled == true;
+        var testSkippingEnabled = testOptimizationSettings.TestsSkippingEnabled == true;
+        var earlyFlakeDetectionEnabled = testOptimizationSettings.EarlyFlakeDetectionEnabled == true;
+        var flakyRetryEnabled = testOptimizationSettings.FlakyRetryEnabled == true;
 
         var hasEvpProxy = !string.IsNullOrEmpty(agentConfiguration?.EventPlatformProxyEndpoint);
         if (agentless || hasEvpProxy)
         {
             // Initialize CI Visibility with the current settings
             Log.Debug("RunCiCommand: Initialize CI Visibility for the runner.");
-            CIVisibility.InitializeFromRunner(ciVisibilitySettings, discoveryService, hasEvpProxy);
+            testOptimization.InitializeFromRunner(testOptimizationSettings, discoveryService, hasEvpProxy);
 
             // Upload git metadata by default (unless is disabled explicitly) or if ITR is enabled (required).
             Log.Debug("RunCiCommand: Uploading repository changes.");
@@ -135,15 +140,15 @@ public static async Task<InitResults> InitializeCiCommandsAsync(
                 ciValues.GitSearchFolder = null;
             }
 
-            var lazyItrClient = new Lazy<IntelligentTestRunnerClient>(() => new(ciValues.WorkspacePath, ciVisibilitySettings));
-            if (ciVisibilitySettings.GitUploadEnabled != false || ciVisibilitySettings.IntelligentTestRunnerEnabled)
+            var client = TestOptimizationClient.Create(ciValues.WorkspacePath, testOptimizationSettings);
+            if (testOptimizationSettings.GitUploadEnabled != false || testOptimizationSettings.IntelligentTestRunnerEnabled)
             {
                 // If we are in git upload only then we can defer the await until the child command exits.
                 async Task UploadRepositoryChangesAsync()
                 {
                     try
                     {
-                        await lazyItrClient.Value.UploadRepositoryChangesAsync().ConfigureAwait(false);
+                        await client.UploadRepositoryChangesAsync().ConfigureAwait(false);
                     }
                     catch (Exception ex)
                     {
@@ -164,32 +169,32 @@ async Task UploadRepositoryChangesAsync()
                 // EVP proxy is enabled.
                 // By setting the environment variables we avoid the usage of the DiscoveryService in each child process
                 // to ask for EVP proxy support.
-                var evpProxyMode = CIVisibility.EventPlatformProxySupportFromEndpointUrl(agentConfiguration?.EventPlatformProxyEndpoint).ToString();
+                var evpProxyMode = testOptimization.TracerManagement?.EventPlatformProxySupportFromEndpointUrl(agentConfiguration?.EventPlatformProxyEndpoint).ToString();
                 profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.ForceAgentsEvpProxy] = evpProxyMode;
                 EnvironmentHelpers.SetEnvironmentVariable(Configuration.ConfigurationKeys.CIVisibility.ForceAgentsEvpProxy, evpProxyMode);
                 Log.Debug("RunCiCommand: EVP proxy was detected: {Mode}", evpProxyMode);
             }
 
             // If we have an api key, and the code coverage or the tests skippable environment variables
             // are not set when the intelligent test runner is enabled, we query the settings api to check if it should enable coverage or not.
-            if (!ciVisibilitySettings.IntelligentTestRunnerEnabled)
+            if (!testOptimizationSettings.IntelligentTestRunnerEnabled)
             {
                 Log.Debug("RunCiCommand: Intelligent test runner is disabled, call to configuration api skipped.");
             }
 
             // If we still don't know if we have to enable code coverage or test skipping, then let's request the configuration API
-            if (ciVisibilitySettings.IntelligentTestRunnerEnabled
-             && (ciVisibilitySettings.CodeCoverageEnabled == null ||
-                 ciVisibilitySettings.TestsSkippingEnabled == null ||
-                 ciVisibilitySettings.EarlyFlakeDetectionEnabled == null ||
-                 ciVisibilitySettings.FlakyRetryEnabled == null))
+            if (testOptimizationSettings.IntelligentTestRunnerEnabled
+             && (testOptimizationSettings.CodeCoverageEnabled == null ||
+                 testOptimizationSettings.TestsSkippingEnabled == null ||
+                 testOptimizationSettings.EarlyFlakeDetectionEnabled == null ||
+                 testOptimizationSettings.FlakyRetryEnabled == null))
             {
                 try
                 {
-                    CIVisibility.Log.Debug("RunCiCommand: Calling configuration api...");
+                    testOptimization.Log.Debug("RunCiCommand: Calling configuration api...");
 
                     // we skip the framework info because we are interested in the target projects info not the runner one.
-                    var itrSettings = await lazyItrClient.Value.GetSettingsAsync(skipFrameworkInfo: true).ConfigureAwait(false);
+                    var itrSettings = await client.GetSettingsAsync(skipFrameworkInfo: true).ConfigureAwait(false);
 
                     // we check if the backend require the git metadata first
                     if (itrSettings.RequireGit == true)
@@ -198,7 +203,7 @@ async Task UploadRepositoryChangesAsync()
                         await uploadRepositoryChangesTask.ConfigureAwait(false);
 
                         Log.Debug("RunCiCommand: calling the configuration api again.");
-                        itrSettings = await lazyItrClient.Value.GetSettingsAsync(skipFrameworkInfo: true).ConfigureAwait(false);
+                        itrSettings = await client.GetSettingsAsync(skipFrameworkInfo: true).ConfigureAwait(false);
                     }
 
                     codeCoverageEnabled = codeCoverageEnabled || itrSettings.CodeCoverage == true || itrSettings.TestsSkipping == true;
@@ -208,7 +213,7 @@ async Task UploadRepositoryChangesAsync()
                 }
                 catch (Exception ex)
                 {
-                    CIVisibility.Log.Warning(ex, "Error getting ITR settings from configuration api");
+                    testOptimization.Log.Warning(ex, "Error getting ITR settings from configuration api");
                 }
             }
         }
@@ -217,9 +222,9 @@ async Task UploadRepositoryChangesAsync()
         Log.Debug("RunCiCommand: TestSkippingEnabled = {Value}", testSkippingEnabled);
         Log.Debug("RunCiCommand: EarlyFlakeDetectionEnabled = {Value}", earlyFlakeDetectionEnabled);
         Log.Debug("RunCiCommand: FlakyRetryEnabled = {Value}", flakyRetryEnabled);
-        ciVisibilitySettings.SetCodeCoverageEnabled(codeCoverageEnabled);
-        ciVisibilitySettings.SetEarlyFlakeDetectionEnabled(earlyFlakeDetectionEnabled);
-        ciVisibilitySettings.SetFlakyRetryEnabled(flakyRetryEnabled);
+        testOptimizationSettings.SetCodeCoverageEnabled(codeCoverageEnabled);
+        testOptimizationSettings.SetEarlyFlakeDetectionEnabled(earlyFlakeDetectionEnabled);
+        testOptimizationSettings.SetFlakyRetryEnabled(flakyRetryEnabled);
         profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.CodeCoverage] = codeCoverageEnabled ? "1" : "0";
         profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.EarlyFlakeDetectionEnabled] = earlyFlakeDetectionEnabled ? "1" : "0";
         profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.FlakyRetryEnabled] = flakyRetryEnabled ? "1" : "0";
@@ -228,7 +233,7 @@ async Task UploadRepositoryChangesAsync()
         {
             // If test skipping is disabled we set this to the child process so we avoid to query the settings api again.
             // If is not disabled we need to query the backend again in the child process with more runtime info.
-            ciVisibilitySettings.SetTestsSkippingEnabled(testSkippingEnabled);
+            testOptimizationSettings.SetTestsSkippingEnabled(testSkippingEnabled);
             profilerEnvironmentVariables[Configuration.ConfigurationKeys.CIVisibility.TestsSkippingEnabled] = "0";
         }