Make frontend drain traffic time configurable (#3934)

* Make frontend drain traffic time configurable

Author: yux0

common/dynamicconfig/constants.go

@@ -202,6 +202,8 @@ const (
 	FrontendThrottledLogRPS = "frontend.throttledLogRPS"
 	// FrontendShutdownDrainDuration is the duration of traffic drain during shutdown
 	FrontendShutdownDrainDuration = "frontend.shutdownDrainDuration"
+	// FrontendShutdownFailHealthcheckDuration is the duration of shutdown failure detection
+	FrontendShutdownFailHealthcheckDuration = "frontend.shutdownFailHealthcheckDuration"
 	// FrontendMaxBadBinaries is the max number of bad binaries in namespace config
 	FrontendMaxBadBinaries = "frontend.maxBadBinaries"
 	// SendRawWorkflowHistory is whether to enable raw history retrieving

service/frontend/service.go

@@ -85,6 +85,7 @@ type Config struct {
 	WorkerBuildIdSizeLimit                 dynamicconfig.IntPropertyFn
 	DisallowQuery                          dynamicconfig.BoolPropertyFnWithNamespaceFilter
 	ShutdownDrainDuration                  dynamicconfig.DurationPropertyFn
+	ShutdownFailHealthcheckDuration        dynamicconfig.DurationPropertyFn
 
 	MaxBadBinaries dynamicconfig.IntPropertyFnWithNamespaceFilter
 
@@ -207,6 +208,7 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, enableReadF
 		BlobSizeLimitWarn:                      dc.GetIntPropertyFilteredByNamespace(dynamicconfig.BlobSizeLimitWarn, 256*1024),
 		ThrottledLogRPS:                        dc.GetIntProperty(dynamicconfig.FrontendThrottledLogRPS, 20),
 		ShutdownDrainDuration:                  dc.GetDurationProperty(dynamicconfig.FrontendShutdownDrainDuration, 0*time.Second),
+		ShutdownFailHealthcheckDuration:        dc.GetDurationProperty(dynamicconfig.FrontendShutdownFailHealthcheckDuration, 10*time.Second),
 		EnableNamespaceNotActiveAutoForwarding: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.EnableNamespaceNotActiveAutoForwarding, true),
 		SearchAttributesNumberOfKeysLimit:      dc.GetIntPropertyFilteredByNamespace(dynamicconfig.SearchAttributesNumberOfKeysLimit, 100),
 		SearchAttributesSizeOfValueLimit:       dc.GetIntPropertyFilteredByNamespace(dynamicconfig.SearchAttributesSizeOfValueLimit, 2*1024),
@@ -336,11 +338,11 @@ func (s *Service) Stop() {
 	// 1. Fail rpc health check, this will cause client side load balancer to stop forwarding requests to this node
 	// 2. wait for failure detection time
 	// 3. stop taking new requests by returning InternalServiceError
-	// 4. Wait for a second
+	// 4. Wait for X second
 	// 5. Stop everything forcefully and return
 
-	requestDrainTime := util.Min(time.Second, s.config.ShutdownDrainDuration())
-	failureDetectionTime := util.Max(0, s.config.ShutdownDrainDuration()-requestDrainTime)
+	requestDrainTime := util.Max(time.Second, s.config.ShutdownDrainDuration())
+	failureDetectionTime := util.Max(0, s.config.ShutdownFailHealthcheckDuration())
 
 	logger.Info("ShutdownHandler: Updating gRPC health status to ShuttingDown")
 	s.healthServer.Shutdown()