Runner config option to disable auto-update. (#1558)

TingluoHuang · thboop · web-flow · commit 8eacbdc79f60 · 2022-01-26T13:23:24.000-05:00
* Runner config option to disable auto-update.

* Update src/Runner.Listener/Configuration/ConfigurationManager.cs

Co-authored-by: Thomas Boop &lt;52323235+thboop@users.noreply.github.com&gt;

* Update src/Runner.Listener/Configuration/ConfigurationManager.cs

Co-authored-by: Thomas Boop &lt;52323235+thboop@users.noreply.github.com&gt;

* Update src/Runner.Listener/Configuration/ConfigurationManager.cs

Co-authored-by: Thomas Boop &lt;52323235+thboop@users.noreply.github.com&gt;

* Update src/Runner.Listener/Configuration/ConfigurationManager.cs

Co-authored-by: Thomas Boop &lt;52323235+thboop@users.noreply.github.com&gt;

* feedback.

Co-authored-by: Thomas Boop &lt;52323235+thboop@users.noreply.github.com&gt;
diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs
@@ -33,6 +33,9 @@ public sealed class RunnerSettings
         [DataMember(EmitDefaultValue = false)]
         public string PoolName { get; set; }
 
+        [DataMember(EmitDefaultValue = false)]
+        public bool DisableUpdate { get; set; }
+
         [DataMember(EmitDefaultValue = false)]
         public bool Ephemeral { get; set; }
 
diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs
@@ -129,6 +129,7 @@ public static class Flags
                     public static readonly string Ephemeral = "ephemeral";
                     public static readonly string Help = "help";
                     public static readonly string Replace = "replace";
+                    public static readonly string DisableUpdate = "disableupdate";
                     public static readonly string Once = "once"; // Keep this around since customers still relies on it
                     public static readonly string RunAsService = "runasservice";
                     public static readonly string Unattended = "unattended";
diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs
@@ -29,6 +29,7 @@ public sealed class CommandSettings
         {
             Constants.Runner.CommandLine.Flags.Check,
             Constants.Runner.CommandLine.Flags.Commit,
+            Constants.Runner.CommandLine.Flags.DisableUpdate,
             Constants.Runner.CommandLine.Flags.Ephemeral,
             Constants.Runner.CommandLine.Flags.Help,
             Constants.Runner.CommandLine.Flags.Once,
@@ -68,6 +69,7 @@ public sealed class CommandSettings
         public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended);
         public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version);
         public bool Ephemeral => TestFlag(Constants.Runner.CommandLine.Flags.Ephemeral);
+        public bool DisableUpdate => TestFlag(Constants.Runner.CommandLine.Flags.DisableUpdate);
 
         // Keep this around since customers still relies on it
         public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once);
diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs
@@ -196,6 +196,7 @@ public async Task ConfigureAsync(CommandSettings command)
             TaskAgent agent;
             while (true)
             {
+                runnerSettings.DisableUpdate = command.DisableUpdate;
                 runnerSettings.Ephemeral = command.Ephemeral;
                 runnerSettings.AgentName = command.GetRunnerName();
 
@@ -213,11 +214,22 @@ public async Task ConfigureAsync(CommandSettings command)
                     if (command.GetReplace())
                     {
                         // Update existing agent with new PublicKey, agent version.
-                        agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral);
+                        agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate);
 
                         try
                         {
                             agent = await _runnerServer.ReplaceAgentAsync(runnerSettings.PoolId, agent);
+                            if (command.DisableUpdate &&
+                                command.DisableUpdate != agent.DisableUpdate)
+                            {
+                                throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'DisableUpdate' flag.");
+                            }
+                            if (command.Ephemeral &&
+                                command.Ephemeral != agent.Ephemeral)
+                            {
+                                throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'Ephemeral' flag.");
+                            }
+
                             _term.WriteSuccessMessage("Successfully replaced the runner");
                             break;
                         }
@@ -236,11 +248,22 @@ public async Task ConfigureAsync(CommandSettings command)
                 else
                 {
                     // Create a new agent.
-                    agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral);
+                    agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate);
 
                     try
                     {
                         agent = await _runnerServer.AddAgentAsync(runnerSettings.PoolId, agent);
+                        if (command.DisableUpdate &&
+                            command.DisableUpdate != agent.DisableUpdate)
+                        {
+                            throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'DisableUpdate' flag.");
+                        }
+                        if (command.Ephemeral &&
+                            command.Ephemeral != agent.Ephemeral)
+                        {
+                            throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'Ephemeral' flag.");
+                        }
+
                         _term.WriteSuccessMessage("Runner successfully added");
                         break;
                     }
@@ -466,7 +489,7 @@ private ICredentialProvider GetCredentialProvider(CommandSettings command, strin
         }
 
 
-        private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet<string> userLabels, bool ephemeral)
+        private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet<string> userLabels, bool ephemeral, bool disableUpdate)
         {
             ArgUtil.NotNull(agent, nameof(agent));
             agent.Authorization = new TaskAgentAuthorization
@@ -478,6 +501,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey,
             agent.Version = BuildConstants.RunnerPackage.Version;
             agent.OSDescription = RuntimeInformation.OSDescription;
             agent.Ephemeral = ephemeral;
+            agent.DisableUpdate = disableUpdate;
             agent.MaxParallelism = 1;
 
             agent.Labels.Clear();
@@ -494,7 +518,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey,
             return agent;
         }
 
-        private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet<string> userLabels, bool ephemeral)
+        private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet<string> userLabels, bool ephemeral, bool disableUpdate)
         {
             TaskAgent agent = new TaskAgent(agentName)
             {
@@ -506,6 +530,7 @@ private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet
                 Version = BuildConstants.RunnerPackage.Version,
                 OSDescription = RuntimeInformation.OSDescription,
                 Ephemeral = ephemeral,
+                DisableUpdate = disableUpdate
             };
 
             agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System));
diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs
@@ -540,6 +540,7 @@ private void PrintUsage(CommandSettings command)
  --work string          Relative runner work directory (default {Constants.Path.WorkDirectory})
  --replace              Replace any existing runner with the same name (default false)
  --pat                  GitHub personal access token used for checking network connectivity when executing `.{separator}run.{ext} --check`
+ --disableupdate        Disable self-hosted runner automatic update to the latest released version`
  --ephemeral            Configure the runner to only take one job and then let the service un-configure the runner after the job finishes (default false)");
 
 #if OS_WINDOWS
diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs
@@ -1,18 +1,18 @@
-﻿using GitHub.DistributedTask.WebApi;
-using Pipelines = GitHub.DistributedTask.Pipelines;
-using GitHub.Runner.Common.Util;
-using GitHub.Services.Common;
-using GitHub.Services.WebApi;
-using System;
+﻿using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Net.Http;
+using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+using GitHub.Services.WebApi;
+using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
 {
@@ -25,6 +25,7 @@ public interface IJobRunner : IRunnerService
     public sealed class JobRunner : RunnerService, IJobRunner
     {
         private IJobServerQueue _jobServerQueue;
+        private RunnerSettings _runnerSettings;
         private ITempDirectoryManager _tempDirectoryManager;
 
         public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
@@ -108,8 +109,8 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
                 jobContext.SetRunnerContext("os", VarUtil.OS);
                 jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);
 
-                var runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
-                jobContext.SetRunnerContext("name", runnerSettings.AgentName);
+                _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
+                jobContext.SetRunnerContext("name", _runnerSettings.AgentName);
 
                 string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
                 Directory.CreateDirectory(toolsDirectory);
@@ -209,6 +210,53 @@ private async Task<TaskResult> CompleteJobAsync(IJobServer jobServer, IExecution
             jobContext.Debug($"Finishing: {message.JobDisplayName}");
             TaskResult result = jobContext.Complete(taskResult);
 
+            if (_runnerSettings.DisableUpdate == true)
+            {
+                try
+                {
+                    var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version);
+                    ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+                    VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection);
+
+                    var runnerServer = HostContext.GetService<IRunnerServer>();
+                    await runnerServer.ConnectAsync(systemConnection.Url, serverCredential);
+                    var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None);
+                    if (serverPackages.Count > 0)
+                    {
+                        serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList();
+                        Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}");
+
+                        var warnOnFailedJob = false; // any minor/patch version behind.
+                        var warnOnOldRunnerVersion = false; // >= 2 minor version behind
+                        if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0))
+                        {
+                            Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}.");
+                            warnOnFailedJob = true;
+                        }
+
+                        if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1)
+                        {
+                            Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}.");
+                            warnOnOldRunnerVersion = true;
+                        }
+
+                        if (result == TaskResult.Failed && warnOnFailedJob)
+                        {
+                            jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}");
+                        }
+                        else if (warnOnOldRunnerVersion)
+                        {
+                            jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}");
+                        }
+                    }
+                }
+                catch (Exception ex)
+                {
+                    // Ignore any error since suggest runner update is best effort.
+                    Trace.Error($"Caught exception during runner version check: {ex}");
+                }
+            }
+
             try
             {
                 await ShutdownQueue(throwOnFailure: true);
diff --git a/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs b/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs
@@ -25,6 +25,7 @@ protected TaskAgentReference(TaskAgentReference referenceToBeCloned)
             this.ProvisioningState = referenceToBeCloned.ProvisioningState;
             this.AccessPoint = referenceToBeCloned.AccessPoint;
             this.Ephemeral = referenceToBeCloned.Ephemeral;
+            this.DisableUpdate = referenceToBeCloned.DisableUpdate;
 
             if (referenceToBeCloned.m_links != null)
             {
@@ -92,6 +93,16 @@ public bool? Ephemeral
             set;
         }
 
+        /// <summary>
+        /// Whether or not this agent should auto-update to latest version.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public bool? DisableUpdate
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Whether or not the agent is online.
         /// </summary>
diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs
@@ -66,7 +66,7 @@ public ConfigurationManagerL0()
             _serviceControlManager = new Mock<ILinuxServiceControlManager>();
 #endif
 
-            var expectedAgent = new TaskAgent(_expectedAgentName) { Id = 1 };
+            var expectedAgent = new TaskAgent(_expectedAgentName) { Id = 1, Ephemeral = true, DisableUpdate = true };
             expectedAgent.Authorization = new TaskAgentAuthorization
             {
                 ClientId = Guid.NewGuid(),
@@ -154,7 +154,7 @@ public async Task CanEnsureConfigure()
                     tc,
                     new[]
                     {
-                       "configure",                
+                       "configure",
                        "--url", _expectedServerUrl,
                        "--name", _expectedAgentName,
                        "--runnergroup", _secondRunnerGroupName,
@@ -163,6 +163,8 @@ public async Task CanEnsureConfigure()
                        "--token", _expectedToken,
                        "--labels", userLabels,
                        "--ephemeral",
+                       "--disableupdate",
+                       "--unattended",
                     });
                 trace.Info("Constructed.");
                 _store.Setup(x => x.IsConfigured()).Returns(false);
@@ -185,7 +187,7 @@ public async Task CanEnsureConfigure()
                 // validate GetAgentPoolsAsync gets called twice with automation pool type
                 _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny<string>(), It.Is<TaskAgentPoolType>(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2));
 
-                var expectedLabels = new List<string>() { "self-hosted", VarUtil.OS, VarUtil.OSArchitecture};
+                var expectedLabels = new List<string>() { "self-hosted", VarUtil.OS, VarUtil.OSArchitecture };
                 expectedLabels.AddRange(userLabels.Split(",").ToList());
 
                 _runnerServer.Verify(x => x.AddAgentAsync(It.IsAny<int>(), It.Is<TaskAgent>(a => a.Labels.Select(x => x.Name).ToHashSet().SetEquals(expectedLabels))), Times.Once);
