Skip to content

Commit 4e7b392

Browse files
jcouvjcouv
authored
Run EndToEnd tests in a more isolated/controlled environment (dotnet#62433)
1 parent 683d177 commit 4e7b392

14 files changed

+86
-18
lines changed

‎Compilers.sln

+7
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "VbcCommandLine", "src\Compi
160160
EndProject
161161
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vbc-arm64", "src\Compilers\VisualBasic\vbc\arm64\vbc-arm64.csproj", "{48C93F90-8776-4847-96D8-127B896D6C80}"
162162
EndProject
163+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests", "src\Compilers\CSharp\Test\EndToEnd\Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj", "{F3D9264A-7CAE-4265-AF48-0C863301F51E}"
164+
EndProject
163165
Global
164166
GlobalSection(SolutionConfigurationPlatforms) = preSolution
165167
Debug|Any CPU = Debug|Any CPU
@@ -410,6 +412,10 @@ Global
410412
{48C93F90-8776-4847-96D8-127B896D6C80}.Debug|Any CPU.Build.0 = Debug|Any CPU
411413
{48C93F90-8776-4847-96D8-127B896D6C80}.Release|Any CPU.ActiveCfg = Release|Any CPU
412414
{48C93F90-8776-4847-96D8-127B896D6C80}.Release|Any CPU.Build.0 = Release|Any CPU
415+
{F3D9264A-7CAE-4265-AF48-0C863301F51E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
416+
{F3D9264A-7CAE-4265-AF48-0C863301F51E}.Debug|Any CPU.Build.0 = Debug|Any CPU
417+
{F3D9264A-7CAE-4265-AF48-0C863301F51E}.Release|Any CPU.ActiveCfg = Release|Any CPU
418+
{F3D9264A-7CAE-4265-AF48-0C863301F51E}.Release|Any CPU.Build.0 = Release|Any CPU
413419
EndGlobalSection
414420
GlobalSection(SolutionProperties) = preSolution
415421
HideSolutionNode = FALSE
@@ -484,6 +490,7 @@ Global
484490
{869E3B79-4E91-45FD-BA37-56DBD2F34721} = {E35DA3D1-16C0-4318-9187-6B664F12A870}
485491
{810B02AD-2EA5-4422-88AC-B71B8AB0DF0B} = {C65C6143-BED3-46E6-869E-9F0BE6E84C37}
486492
{48C93F90-8776-4847-96D8-127B896D6C80} = {C65C6143-BED3-46E6-869E-9F0BE6E84C37}
493+
{F3D9264A-7CAE-4265-AF48-0C863301F51E} = {32A48625-F0AD-419D-828B-A50BDABA38EA}
487494
EndGlobalSection
488495
GlobalSection(ExtensibilityGlobals) = postSolution
489496
SolutionGuid = {6F599E08-A9EA-4FAA-897F-5D824B0210E6}

‎Compilers.slnf

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"src\\Compilers\\CSharp\\Test\\CommandLine\\Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests.csproj",
88
"src\\Compilers\\CSharp\\Test\\Emit\\Microsoft.CodeAnalysis.CSharp.Emit.UnitTests.csproj",
99
"src\\Compilers\\CSharp\\Test\\Emit2\\Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests.csproj",
10+
"src\\Compilers\\CSharp\\Test\\EndToEnd\\Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj",
1011
"src\\Compilers\\CSharp\\Test\\IOperation\\Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests.csproj",
1112
"src\\Compilers\\CSharp\\Test\\Semantic\\Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests.csproj",
1213
"src\\Compilers\\CSharp\\Test\\Symbol\\Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests.csproj",

‎Roslyn.sln

+7
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Net.Compilers.Too
509509
EndProject
510510
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Net.Compilers.Toolset.Package.Arm64", "src\NuGet\Microsoft.Net.Compilers.Toolset\arm64\Microsoft.Net.Compilers.Toolset.Package.Arm64.csproj", "{A9A8ADE5-F123-4109-9FA4-4B92F1657043}"
511511
EndProject
512+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests", "src\Compilers\CSharp\Test\EndToEnd\Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj", "{C247414A-8946-4BAB-BE1F-C82B90C63EF6}"
513+
EndProject
512514
Global
513515
GlobalSection(SolutionConfigurationPlatforms) = preSolution
514516
Debug|Any CPU = Debug|Any CPU
@@ -1235,6 +1237,10 @@ Global
12351237
{A9A8ADE5-F123-4109-9FA4-4B92F1657043}.Debug|Any CPU.Build.0 = Debug|Any CPU
12361238
{A9A8ADE5-F123-4109-9FA4-4B92F1657043}.Release|Any CPU.ActiveCfg = Release|Any CPU
12371239
{A9A8ADE5-F123-4109-9FA4-4B92F1657043}.Release|Any CPU.Build.0 = Release|Any CPU
1240+
{C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1241+
{C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Debug|Any CPU.Build.0 = Debug|Any CPU
1242+
{C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.ActiveCfg = Release|Any CPU
1243+
{C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.Build.0 = Release|Any CPU
12381244
EndGlobalSection
12391245
GlobalSection(SolutionProperties) = preSolution
12401246
HideSolutionNode = FALSE
@@ -1466,6 +1472,7 @@ Global
14661472
{46429C41-0733-4568-9BA6-3F52CB47C4ED} = {BE25E872-1667-4649-9D19-96B83E75A44E}
14671473
{6131713D-DFB4-49B5-8010-50071FED3E85} = {C52D8057-43AF-40E6-A01B-6CDBB7301985}
14681474
{A9A8ADE5-F123-4109-9FA4-4B92F1657043} = {C52D8057-43AF-40E6-A01B-6CDBB7301985}
1475+
{C247414A-8946-4BAB-BE1F-C82B90C63EF6} = {32A48625-F0AD-419D-828B-A50BDABA38EA}
14691476
EndGlobalSection
14701477
GlobalSection(ExtensibilityGlobals) = postSolution
14711478
SolutionGuid = {604E6B91-7BC0-4126-AE07-D4D2FEFC3D29}

‎src/Compilers/CSharp/Portable/Microsoft.CodeAnalysis.CSharp.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
6363
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
6464
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
65+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
6566
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests" />
6667
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests" />
6768
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />

‎src/Compilers/CSharp/Test/Emit/Emit/EndToEndTests.cs ‎src/Compilers/CSharp/Test/EndToEnd/EndToEndTests.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@
1313
using Microsoft.CodeAnalysis.CSharp.Symbols;
1414
using Microsoft.CodeAnalysis.PooledObjects;
1515

16-
namespace Microsoft.CodeAnalysis.CSharp.UnitTests.Emit
16+
namespace Microsoft.CodeAnalysis.CSharp.UnitTests.EndToEnd
1717
{
18+
[TestCaseOrderer("XUnit.Project.Orderers.AlphabeticalOrderer", "XUnit.Project")]
1819
public class EndToEndTests : EmitMetadataTestBase
1920
{
2021
/// <summary>

‎src/Compilers/CSharp/Test/EndToEnd/Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
3+
<Project Sdk="Microsoft.NET.Sdk">
4+
<PropertyGroup>
5+
<OutputType>Library</OutputType>
6+
<RootNamespace>Microsoft.CodeAnalysis.CSharp.UnitTests</RootNamespace>
7+
<TargetFrameworks>net6.0;net472</TargetFrameworks>
8+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
9+
</PropertyGroup>
10+
<ItemGroup Label="Project References">
11+
<ProjectReference Include="..\..\..\..\Test\PdbUtilities\Roslyn.Test.PdbUtilities.csproj" />
12+
<ProjectReference Include="..\..\..\Test\Core\Microsoft.CodeAnalysis.Test.Utilities.csproj" />
13+
<ProjectReference Include="..\..\..\Core\Portable\Microsoft.CodeAnalysis.csproj" />
14+
<ProjectReference Include="..\..\..\Test\Resources\Core\Microsoft.CodeAnalysis.Compiler.Test.Resources.csproj" />
15+
<ProjectReference Include="..\..\..\Test\Utilities\CSharp\Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj" />
16+
<ProjectReference Include="..\..\..\VisualBasic\Portable\Microsoft.CodeAnalysis.VisualBasic.vbproj" />
17+
<ProjectReference Include="..\..\Portable\Microsoft.CodeAnalysis.CSharp.csproj" />
18+
<Compile Include="..\..\..\Core\MSBuildTask\MvidReader.cs">
19+
<Link>Emit\MvidReader.cs</Link>
20+
</Compile>
21+
<PackageReference Include="Microsoft.CSharp" Version="$(MicrosoftCSharpVersion)" />
22+
<PackageReference Include="Microsoft.DiaSymReader" Version="$(MicrosoftDiaSymReaderVersion)" />
23+
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
24+
</ItemGroup>
25+
<Import Project="$(RepositoryEngineeringDir)targets\ILAsm.targets" />
26+
</Project>

‎src/Compilers/CSharp/Test/EndToEnd/README.md

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
The EndToEnd tests are isolated from other compiler test to reduce variability.
2+
3+
Specifically, other tests could:
4+
1. affect JIT ordering,
5+
2. cause GC side-effects,
6+
3. affect implicit caching,
7+
4. change the starting stack size.
8+
9+
Because EndToEnd tests are in a single test class, they also don't get parallelized.

‎src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
6363
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
6464
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
65+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
6566
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests" />
6667
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests" />
6768
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />

‎src/Compilers/Test/Core/Microsoft.CodeAnalysis.Test.Utilities.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
1818
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
1919
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
20+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
2021
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests" />
2122
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests" />
2223
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />

‎src/Compilers/Test/Utilities/CSharp/Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
2323
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
2424
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
25+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
2526
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests" />
2627
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.IOperation.UnitTests" />
2728
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />

‎src/EditorFeatures/TestUtilities/Microsoft.CodeAnalysis.EditorFeatures.Test.Utilities.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
5656
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
5757
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
58+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
5859
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />
5960
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests" />
6061
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Syntax.UnitTests" />

‎src/Scripting/CSharp/Microsoft.CodeAnalysis.CSharp.Scripting.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EditorFeatures" WorkItem="https://github.com/dotnet/roslyn/issues/5661"/>
2626
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
2727
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
28+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
2829
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.UnitTests" />
2930
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.VisualBasic.Emit.UnitTests" />
3031
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.Scripting.UnitTests" />

‎src/Test/PdbUtilities/Roslyn.Test.PdbUtilities.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests" />
1313
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit.UnitTests" />
1414
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Emit2.UnitTests" />
15+
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests" />
1516
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.WinRT.UnitTests" />
1617
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Semantic.UnitTests" />
1718
<InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Symbol.UnitTests" />

‎src/Tools/Source/RunTests/Program.cs

+27-17
Original file line numberDiff line numberDiff line change
@@ -301,28 +301,12 @@ private static List<AssemblyInfo> GetAssemblyList(Options options)
301301
foreach (var project in Directory.EnumerateDirectories(binDirectory, "*", SearchOption.TopDirectoryOnly))
302302
{
303303
var name = Path.GetFileName(project);
304-
var include = false;
305-
foreach (var pattern in options.IncludeFilter)
306-
{
307-
if (Regex.IsMatch(name, pattern.Trim('\'', '"')))
308-
{
309-
include = true;
310-
}
311-
}
312304

313-
if (!include)
305+
if (!shouldInclude(name, options) || shouldExclude(name, options))
314306
{
315307
continue;
316308
}
317309

318-
foreach (var pattern in options.ExcludeFilter)
319-
{
320-
if (Regex.IsMatch(name, pattern.Trim('\'', '"')))
321-
{
322-
continue;
323-
}
324-
}
325-
326310
var fileName = $"{name}.dll";
327311
foreach (var targetFramework in options.TargetFrameworks)
328312
{
@@ -348,6 +332,32 @@ private static List<AssemblyInfo> GetAssemblyList(Options options)
348332
}
349333

350334
return list;
335+
336+
static bool shouldInclude(string name, Options options)
337+
{
338+
foreach (var pattern in options.IncludeFilter)
339+
{
340+
if (Regex.IsMatch(name, pattern.Trim('\'', '"')))
341+
{
342+
return true;
343+
}
344+
}
345+
346+
return false;
347+
}
348+
349+
static bool shouldExclude(string name, Options options)
350+
{
351+
foreach (var pattern in options.ExcludeFilter)
352+
{
353+
if (Regex.IsMatch(name, pattern.Trim('\'', '"')))
354+
{
355+
return true;
356+
}
357+
}
358+
359+
return false;
360+
}
351361
}
352362

353363
private static void DisplayResults(Display display, ImmutableArray<TestResult> testResults)

0 commit comments

Comments
 (0)