Integrate the initial node metrics (#11481)

Author: JanProvaznik

src/Build.UnitTests/TelemetryTests.cs

@@ -4,8 +4,11 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.Json;
 using Microsoft.Build.Execution;
 using Microsoft.Build.Framework;
+using Microsoft.Build.Framework.Telemetry;
+using Microsoft.Build.TelemetryInfra;
 using Microsoft.Build.UnitTests;
 using Shouldly;
 using Xunit;
@@ -165,5 +168,30 @@ public void WorkerNodeTelemetryCollection_CustomTargetsAndTasks()
 
             workerNodeTelemetryData.TasksExecutionData.Keys.ShouldAllBe(k => !k.IsFromNugetCache);
         }
+
+        [Fact]
+        public void Foo()
+        {
+            WorkerNodeTelemetryData wd = new WorkerNodeTelemetryData(
+                new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>()
+                {
+                    {
+                        new TaskOrTargetTelemetryKey("TaskA", false, true),
+                        new TaskExecutionStats(TimeSpan.FromSeconds(2.1554548), 5, 545)
+                    },
+                    {
+                        new TaskOrTargetTelemetryKey("TaskA", true, false),
+                        new TaskExecutionStats(TimeSpan.FromSeconds(254548), 6, 54545451)
+                    },
+                },
+                new Dictionary<TaskOrTargetTelemetryKey, bool>()
+                {
+                    { new TaskOrTargetTelemetryKey("TargetA", false, true, false), false },
+                    { new TaskOrTargetTelemetryKey("TargetA", true, true, false), false },
+                    { new TaskOrTargetTelemetryKey("TargetB", false, false, true), false }
+                });
+
+            var holder = TelemetryDataUtils.AsActivityDataHolder(wd, true, true);
+        }
     }
 }

src/Build/BackEnd/BuildManager/BuildManager.cs

@@ -36,6 +36,7 @@
 using Microsoft.Build.Logging;
 using Microsoft.Build.Shared;
 using Microsoft.Build.Shared.Debugging;
+using Microsoft.Build.TelemetryInfra;
 using Microsoft.NET.StringTools;
 using ExceptionHandling = Microsoft.Build.Shared.ExceptionHandling;
 using ForwardingLoggerRecord = Microsoft.Build.Logging.ForwardingLoggerRecord;
@@ -257,6 +258,11 @@ public class BuildManager : INodePacketHandler, IBuildComponentHost, IDisposable
         /// </summary>
         private BuildTelemetry? _buildTelemetry;
 
+        /// <summary>
+        /// Logger, that if instantiated - will receive and expose telemetry data from worker nodes.
+        /// </summary>
+        private InternalTelemetryConsumingLogger? _telemetryConsumingLogger;
+
         private ProjectCacheService? _projectCacheService;
 
         private bool _hasProjectCacheServiceInitializedVsScenario;
@@ -563,6 +569,7 @@ public void BeginBuild(BuildParameters parameters)
                 // Initialize components.
                 _nodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.NodeManager) as INodeManager;
 
+                _buildParameters.IsTelemetryEnabled |= OpenTelemetryManager.Instance.IsActive();
                 var loggingService = InitializeLoggingService();
 
                 // Log deferred messages and response files
@@ -1134,7 +1141,10 @@ private void EndBuildTelemetry()
         {
             OpenTelemetryManager.Instance.DefaultActivitySource?
                 .StartActivity("Build")?
-                .WithTags(_buildTelemetry!)
+                .WithTags(_buildTelemetry)
+                .WithTags(_telemetryConsumingLogger?.WorkerNodeTelemetryData.AsActivityDataHolder(
+                    includeTasksDetails: !Traits.Instance.ExcludeTasksDetailsFromTelemetry,
+                    includeTargetDetails: false))
                 .WithStartTime(_buildTelemetry!.InnerStartAt)
                 .Dispose();
             OpenTelemetryManager.Instance.ForceFlush();
@@ -2986,10 +2996,10 @@ private ILoggingService CreateLoggingService(
                     loggerSwitchParameters: null,
                     verbosity: LoggerVerbosity.Quiet);
 
-                ILogger internalTelemetryLogger =
+                _telemetryConsumingLogger =
                     new InternalTelemetryConsumingLogger();
 
-                ForwardingLoggerRecord[] forwardingLogger = { new ForwardingLoggerRecord(internalTelemetryLogger, forwardingLoggerDescription) };
+                ForwardingLoggerRecord[] forwardingLogger = { new ForwardingLoggerRecord(_telemetryConsumingLogger, forwardingLoggerDescription) };
 
                 forwardingLoggers = forwardingLoggers?.Concat(forwardingLogger) ?? forwardingLogger;
             }

src/Build/Microsoft.Build.csproj

@@ -174,7 +174,9 @@
     <Compile Include="Instance\IPropertyElementWithLocation.cs" />
     <Compile Include="Logging\BuildEventArgsExtensions.cs" />
     <Compile Include="Logging\TerminalLogger\**\*.cs" />
+    <Compile Include="TelemetryInfra\InternalTelemetryConsumingLogger.cs" />
     <Compile Include="TelemetryInfra\ITelemetryForwarder.cs" />
+    <Compile Include="TelemetryInfra\TelemetryDataUtils.cs" />
     <Compile Include="TelemetryInfra\TelemetryForwarderProvider.cs" />
     <Compile Include="Utilities\ReaderWriterLockSlimExtensions.cs" />
     <Compile Include="BackEnd\Node\ConsoleOutput.cs" />

src/Framework/Telemetry/InternalTelemetryConsumingLogger.cs src/Build/TelemetryInfra/InternalTelemetryConsumingLogger.cs

@@ -3,8 +3,12 @@
 
 using System;
 using System.Linq;
+using System.Text.Json;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Framework.Telemetry;
+using System.IO;
 
-namespace Microsoft.Build.Framework;
+namespace Microsoft.Build.TelemetryInfra;
 
 internal sealed class InternalTelemetryConsumingLogger : ILogger
 {
@@ -23,6 +27,8 @@ public void Initialize(IEventSource eventSource)
 
     private readonly WorkerNodeTelemetryData _workerNodeTelemetryData = new();
 
+    public IWorkerNodeTelemetryData WorkerNodeTelemetryData => _workerNodeTelemetryData;
+
     private void EventSource5_WorkerNodeTelemetryLogged(object? sender, WorkerNodeTelemetryEventArgs e)
     {
         _workerNodeTelemetryData.Add(e.WorkerNodeTelemetryData);
@@ -32,11 +38,12 @@ private void EventSourceOnBuildFinished(object sender, BuildFinishedEventArgs e)
     {
         TestOnly_InternalTelemetryAggregted?.Invoke(_workerNodeTelemetryData);
         FlushDataIntoConsoleIfRequested();
+        FlushDataIntoJsonFileIfRequested();
     }
 
     private void FlushDataIntoConsoleIfRequested()
     {
-        if (Environment.GetEnvironmentVariable("MSBUILDOUTPUTNODESTELEMETRY") != "1")
+        if (!Traits.IsEnvVarOneOrTrue("MSBUILDOUTPUTNODESTELEMETRY"))
         {
             return;
         }
@@ -75,6 +82,22 @@ private void FlushDataIntoConsoleIfRequested()
         Console.WriteLine("==========================================");
     }
 
+    private void FlushDataIntoJsonFileIfRequested()
+    {
+        const string jsonFileNameVariable = "MSBUILDNODETELEMETRYFILENAME";
+        var jsonFilePath = Environment.GetEnvironmentVariable(jsonFileNameVariable);
+        if (string.IsNullOrEmpty(jsonFilePath))
+        {
+            return;
+        }
+
+        var telemetryTags = _workerNodeTelemetryData.AsActivityDataHolder(true, true)?.GetActivityProperties();
+
+        using var stream = File.OpenWrite(jsonFilePath);
+        stream.SetLength(0);
+        JsonSerializer.Serialize(stream, telemetryTags, new JsonSerializerOptions() { WriteIndented = true });
+    }
+
     public void Shutdown()
     { }
 }