Randomize the order of tests (#5831)

## Summary of changes

Randomize the order of the tests.

## Reason for change

Flaky tests are much harder to fix when we discover them long after they
have been written. By randomizing the order of the tests, I'm hoping to
make them fail earlier.
In practice, this could temporarily increase the overall flakiness, but
I expect this will reduce the overall effort spent on fixing tests.

## Implementation details

In `CustomTestFramework`, randomize the list of all tests in each
collections, and the collections themselves.
The seed is displayed in the output. When a test order causes tests to
fail, this allows to deterministically reproduce that test order.

## Other details

Four other issues were found thanks to this:
#6535,
#6532,
#6511,
#6509

Author: kevingosse

.azure-pipelines/steps/run-in-docker.yml

@@ -100,6 +100,7 @@ steps:
         --env DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH \
         --env DD_LOGGER_DD_TAGS \
         --env IS_SSI_RUN \
+        --env RANDOM_SEED \
         ${{ parameters.extraArgs }} \
         dd-trace-dotnet/${{ parameters.baseImage }}-${{ parameters.target }}:$sdkVersion \
         dotnet /build/bin/Debug/_build.dll ${{ parameters.command }}

docker-compose.yml

@@ -454,6 +454,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - IS_SSI_RUN
+      - RANDOM_SEED
     hostname: integrationtests
     depends_on:
       - servicestackredis
@@ -524,6 +525,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - IS_SSI_RUN
+      - RANDOM_SEED
     hostname: integrationtests
 
   IntegrationTests.Serverless:
@@ -580,6 +582,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - IS_SSI_RUN
+      - RANDOM_SEED
     hostname: integrationtests
 
   ExplorationTests:
@@ -722,6 +725,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - IS_SSI_RUN
+      - RANDOM_SEED
     depends_on:
       - servicestackredis_arm64
       - stackexchangeredis_arm64
@@ -800,6 +804,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - IS_SSI_RUN
+      - RANDOM_SEED
 
   start-test-agent:
     image: andrewlock/wait-for-dependencies

tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs

@@ -126,12 +126,26 @@ protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus me
                     })
                     .ToList();
 
+                if (Environment.GetEnvironmentVariable("RANDOM_SEED") is not { } environmentSeed
+                    || !int.TryParse(environmentSeed, out var seed))
+                {
+                    seed = new Random().Next();
+                }
+
+                DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using seed {seed} to randomize tests order"));
+
+                var random = new Random(seed);
+                Shuffle(collections, random);
+
+                foreach (var collection in collections)
+                {
+                    Shuffle(collection.TestCases, random);
+                }
+
                 _runTestCollectionsCallback?.Invoke(DiagnosticMessageSink, collections.SelectMany(c => c.TestCases));
 
                 var summary = new RunSummary();
-
                 using var runner = new ConcurrentRunner();
-
                 var tasks = new List<Task<RunSummary>>();
 
                 foreach (var test in collections.Where(t => !t.DisableParallelization))
@@ -172,6 +186,20 @@ private static bool IsParallelizationDisabled(ITestCollection collection)
 
                 return attr?.GetNamedArgument<bool>(nameof(CollectionDefinitionAttribute.DisableParallelization)) is true;
             }
+
+            private static void Shuffle<T>(IList<T> list, Random rng)
+            {
+                int n = list.Count;
+
+                while (n > 1)
+                {
+                    n--;
+                    int k = rng.Next(n + 1);
+                    var value = list[k];
+                    list[k] = list[n];
+                    list[n] = value;
+                }
+            }
         }
 
         private class CustomTestCollectionRunner : XunitTestCollectionRunner