[Release 2.1] Tests | Fix randomly failing tests (#942)

Author: cheenamalhotra

src/Microsoft.Data.SqlClient/tests/FunctionalTests/AlwaysEncryptedTests/ExceptionsAlgorithmErrors.cs

@@ -76,13 +76,19 @@ public class ExceptionCertFixture : IDisposable
 
         public ExceptionCertFixture()
         {
-            certificate = Utility.CreateCertificate();
+            if(certificate == null)
+            {
+                certificate = Utility.CreateCertificate();
+            }
             thumbprint = certificate.Thumbprint;
             certificatePath = string.Format("CurrentUser/My/{0}", thumbprint);
             cek = Utility.GenerateRandomBytes(32);
             encryptedCek = certStoreProvider.EncryptColumnEncryptionKey(certificatePath, "RSA_OAEP", cek);
 #if NET46
-            masterKeyCertificateNPK = Utility.CreateCertificateWithNoPrivateKey();
+            if(masterKeyCertificateNPK == null)
+            {
+                masterKeyCertificateNPK = Utility.CreateCertificateWithNoPrivateKey();
+            }
             thumbprintNPK = masterKeyCertificateNPK.Thumbprint;
             masterKeyPathNPK = "CurrentUser/My/" + thumbprintNPK;
 #endif

src/Microsoft.Data.SqlClient/tests/FunctionalTests/AlwaysEncryptedTests/ExceptionsCertStore.cs

@@ -29,7 +29,7 @@ public class ConversionTests : IDisposable
         private const decimal SmallMoneyMinValue = -214748.3648M;
         private const int MaxLength = 10000;
         private int NumberOfRows = DataTestUtility.EnclaveEnabled ? 10 : 100;
-        private readonly X509Certificate2 certificate;
+        private static X509Certificate2 certificate;
         private ColumnMasterKey columnMasterKey;
         private ColumnEncryptionKey columnEncryptionKey;
         private SqlColumnEncryptionCertificateStoreProvider certStoreProvider = new SqlColumnEncryptionCertificateStoreProvider();
@@ -55,7 +55,10 @@ public ColumnMetaData(SqlDbType columnType, int columnSize, int precision, int s
 
         public ConversionTests()
         {
-            certificate = CertificateUtility.CreateCertificate();
+            if(certificate == null)
+            {
+                certificate = CertificateUtility.CreateCertificate();
+            }
             columnMasterKey = new CspColumnMasterKey(DatabaseHelper.GenerateUniqueName("CMK"), certificate.Thumbprint, certStoreProvider, DataTestUtility.EnclaveEnabled);
             databaseObjects.Add(columnMasterKey);
 

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ConversionTests.cs

@@ -15,7 +15,7 @@ public class SQLSetupStrategy : IDisposable
     {
         internal const string ColumnEncryptionAlgorithmName = @"AEAD_AES_256_CBC_HMAC_SHA256";
 
-        protected internal readonly X509Certificate2 certificate;
+        protected static X509Certificate2 certificate;
         public string keyPath { get; internal set; }
         public Table ApiTestTable { get; private set; }
         public Table BulkCopyAEErrorMessageTestTable { get; private set; }
@@ -56,7 +56,10 @@ public class SQLSetupStrategy : IDisposable
 
         public SQLSetupStrategy()
         {
-            certificate = CertificateUtility.CreateCertificate();
+            if(certificate == null)
+            {
+                certificate = CertificateUtility.CreateCertificate();
+            }
         }
 
         protected SQLSetupStrategy(string customKeyPath) => keyPath = customKeyPath;

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs

@@ -369,7 +369,7 @@ public static void ActiveDirectoryManagedIdentityWithCredentialsMustFail()
             Assert.Contains(expectedMessage, e.Message);
         }
 
-        [ConditionalFact(nameof(IsAADConnStringsSetup))]
+        [ConditionalFact(nameof(IsAADConnStringsSetup), nameof(IsManagedIdentitySetup))]
         public static void ActiveDirectoryManagedIdentityWithPasswordMustFail()
         {
             // connection fails with expected error message.
@@ -385,7 +385,7 @@ public static void ActiveDirectoryManagedIdentityWithPasswordMustFail()
 
         [InlineData("2445343 2343253")]
         [InlineData("2445343$#^@@%2343253")]
-        [ConditionalTheory(nameof(IsAADConnStringsSetup))]
+        [ConditionalTheory(nameof(IsAADConnStringsSetup), nameof(IsManagedIdentitySetup))]
         public static void ActiveDirectoryManagedIdentityWithInvalidUserIdMustFail(string userId)
         {
             // connection fails with expected error message.
@@ -444,33 +444,42 @@ public static void ADInteractiveUsingSSPI()
             ConnectAndDisconnect(connStr);
         }
 
+        // Test passes locally everytime, but in pieplines fails randomly with uncertainity.
+        // e.g. Second AAD connection too slow (802ms)! (More than 30% of the first (576ms).)
+        [ActiveIssue(16058)]
         [ConditionalFact(nameof(IsAADConnStringsSetup))]
         public static void ConnectionSpeed()
         {
+            var connString = DataTestUtility.AADPasswordConnectionString;
+            
             //Ensure server endpoints are warm
-            using (var connectionDrill = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
+            using (var connectionDrill = new SqlConnection(connString))
             {
                 connectionDrill.Open();
             }
+            
             SqlConnection.ClearAllPools();
+            ActiveDirectoryAuthenticationProvider.ClearUserTokenCache();
+
+            Stopwatch firstConnectionTime = new Stopwatch();
+            Stopwatch secondConnectionTime = new Stopwatch();
 
-            using (var connectionDrill = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
+            using (var connectionDrill = new SqlConnection(connString))
             {
-                Stopwatch firstConnectionTime = new Stopwatch();
                 firstConnectionTime.Start();
                 connectionDrill.Open();
                 firstConnectionTime.Stop();
-                using (var connectionDrill2 = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
+                using (var connectionDrill2 = new SqlConnection(connString))
                 {
-                    Stopwatch secondConnectionTime = new Stopwatch();
                     secondConnectionTime.Start();
                     connectionDrill2.Open();
                     secondConnectionTime.Stop();
-                    // Subsequent AAD connections within a short timeframe should use an auth token cached from the connection pool
-                    // Second connection speed in tests was typically 10-15% of the first connection time. Using 30% since speeds may vary.
-                    Assert.True(secondConnectionTime.ElapsedMilliseconds / firstConnectionTime.ElapsedMilliseconds < .30, $"Second AAD connection too slow ({secondConnectionTime.ElapsedMilliseconds}ms)! (More than 30% of the first ({firstConnectionTime.ElapsedMilliseconds}ms).)");
                 }
             }
+            
+            // Subsequent AAD connections within a short timeframe should use an auth token cached from the connection pool
+            // Second connection speed in tests was typically 10-15% of the first connection time. Using 30% since speeds may vary.
+            Assert.True(((double)secondConnectionTime.ElapsedMilliseconds / firstConnectionTime.ElapsedMilliseconds) < 0.30, $"Second AAD connection too slow ({secondConnectionTime.ElapsedMilliseconds}ms)! (More than 30% of the first ({firstConnectionTime.ElapsedMilliseconds}ms).)");
         }
 
         #region Managed Identity Authentication tests

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/AADConnectionTest.cs

@@ -131,11 +131,25 @@ public static void WarningsBeforeRowsTest()
         private static bool CheckThatExceptionsAreDistinctButHaveSameData(SqlException e1, SqlException e2)
         {
             Assert.True(e1 != e2, "FAILED: verification of exception cloning in subsequent connection attempts");
-
             Assert.False((e1 == null) || (e2 == null), "FAILED: One of exceptions is null, another is not");
 
             bool equal = (e1.Message == e2.Message) && (e1.HelpLink == e2.HelpLink) && (e1.InnerException == e2.InnerException)
-                && (e1.Source == e2.Source) && (e1.Data.Count == e2.Data.Count) && (e1.Errors == e2.Errors);
+                && (e1.Source == e2.Source) && (e1.Data.Count == e2.Data.Count) && (e1.Errors.Count == e2.Errors.Count);
+
+            for (int i = 0; i < e1.Errors.Count; i++)
+            {
+                equal = e1.Errors[i].Number == e2.Errors[i].Number
+                    && e1.Errors[i].Message == e2.Errors[i].Message
+                    && e1.Errors[i].LineNumber == e2.Errors[i].LineNumber
+                    && e1.Errors[i].State == e2.Errors[i].State
+                    && e1.Errors[i].Class == e2.Errors[i].Class
+                    && e1.Errors[i].Server == e2.Errors[i].Server
+                    && e1.Errors[i].Procedure == e2.Errors[i].Procedure
+                    && e1.Errors[i].Source == e2.Errors[i].Source;
+                if (!equal)
+                    break;
+            }
+
             IDictionaryEnumerator enum1 = e1.Data.GetEnumerator();
             IDictionaryEnumerator enum2 = e2.Data.GetEnumerator();
             while (equal)
@@ -147,7 +161,6 @@ private static bool CheckThatExceptionsAreDistinctButHaveSameData(SqlException e
             }
 
             Assert.True(equal, string.Format("FAILED: exceptions do not contain the same data (besides call stack):\nFirst: {0}\nSecond: {1}\n", e1, e2));
-
             return true;
         }
 

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ExceptionTest/ExceptionTest.cs

@@ -242,48 +242,63 @@ public void NullTest()
                     p.UdtTypeName = "Utf8String";
                     p.Value = DBNull.Value;
 
-                    using (SqlTransaction trans = conn.BeginTransaction())
+                    bool rerun = false;
+                    do
                     {
-                        com.Transaction = trans;
-                        using (SqlDataReader reader = com.ExecuteReader())
+                        try
                         {
-
-                            Utf8String[] expectedValues = {
-                                new Utf8String("this"),
-                                new Utf8String("is"),
-                                new Utf8String("a"),
-                                new Utf8String("test")
-                            };
-
-                            int currentValue = 0;
-                            do
+                            using (SqlTransaction trans = conn.BeginTransaction())
                             {
-                                while (reader.Read())
+                                com.Transaction = trans;
+                                using (SqlDataReader reader = com.ExecuteReader())
                                 {
-                                    DataTestUtility.AssertEqualsWithDescription(1, reader.FieldCount, "Unexpected FieldCount.");
-                                    if (currentValue < expectedValues.Length)
+                                    Utf8String[] expectedValues = {
+                                        new Utf8String("this"),
+                                        new Utf8String("is"),
+                                        new Utf8String("a"),
+                                        new Utf8String("test")
+                                    };
+
+                                    int currentValue = 0;
+                                    do
                                     {
-                                        DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetValue(0), "Unexpected Value.");
-                                        DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetSqlValue(0), "Unexpected SQL Value.");
-                                    }
-                                    else
-                                    {
-                                        DataTestUtility.AssertEqualsWithDescription(DBNull.Value, reader.GetValue(0), "Unexpected Value.");
-
-                                        Utf8String sqlValue = (Utf8String)reader.GetSqlValue(0);
-                                        INullable iface = sqlValue as INullable;
-                                        Assert.True(iface != null, "Expected interface cast to return a non-null value.");
-                                        Assert.True(iface.IsNull, "Expected interface cast to have IsNull==true.");
+                                        while (reader.Read())
+                                        {
+                                            DataTestUtility.AssertEqualsWithDescription(1, reader.FieldCount, "Unexpected FieldCount.");
+                                            if (currentValue < expectedValues.Length)
+                                            {
+                                                DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetValue(0), "Unexpected Value.");
+                                                DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetSqlValue(0), "Unexpected SQL Value.");
+                                            }
+                                            else
+                                            {
+                                                DataTestUtility.AssertEqualsWithDescription(DBNull.Value, reader.GetValue(0), "Unexpected Value.");
+
+                                                Utf8String sqlValue = (Utf8String)reader.GetSqlValue(0);
+                                                INullable iface = sqlValue as INullable;
+                                                Assert.True(iface != null, "Expected interface cast to return a non-null value.");
+                                                Assert.True(iface.IsNull, "Expected interface cast to have IsNull==true.");
+                                            }
+
+                                            currentValue++;
+                                            Assert.True(currentValue <= (expectedValues.Length + 1), "Expected to only hit one extra result.");
+                                        }
                                     }
-
-                                    currentValue++;
-                                    Assert.True(currentValue <= (expectedValues.Length + 1), "Expected to only hit one extra result.");
+                                    while (reader.NextResult());
+                                    DataTestUtility.AssertEqualsWithDescription(currentValue, (expectedValues.Length + 1), "Did not hit all expected values.");
+                                    rerun = false;
                                 }
                             }
-                            while (reader.NextResult());
-                            DataTestUtility.AssertEqualsWithDescription(currentValue, (expectedValues.Length + 1), "Did not hit all expected values.");
+                        }
+                        catch (SqlException e)
+                        {
+                            if(e.Message.Contains("Rerun the transaction"))
+                                rerun = true;
+                            else 
+                                throw e;
                         }
                     }
+                    while(rerun);
                 }
             }
         }