Add separate dynamic config knobs for internal-frontend rate limiting (#3800)

Author: dnr

common/dynamicconfig/constants.go

@@ -165,7 +165,7 @@ const (
 	FrontendMaxNamespaceRPSPerInstance = "frontend.namespaceRPS"
 	// FrontendMaxNamespaceBurstPerInstance is workflow namespace burst limit
 	FrontendMaxNamespaceBurstPerInstance = "frontend.namespaceBurst"
-	// FrontendMaxNamespaceCountPerInstance is workflow namespace count limit per second
+	// FrontendMaxNamespaceCountPerInstance limits concurrent task queue polls per namespace per instance
 	FrontendMaxNamespaceCountPerInstance = "frontend.namespaceCount"
 	// FrontendMaxNamespaceVisibilityRPSPerInstance is namespace rate limit per second for visibility APIs.
 	// This config is EXPERIMENTAL and may be changed or removed in a later release.
@@ -177,11 +177,18 @@ const (
 	// The limit is evenly distributed among available frontend service instances.
 	// If this is set, it overwrites per instance limit "frontend.namespaceRPS".
 	FrontendGlobalNamespaceRPS = "frontend.globalNamespaceRPS"
+	// InternalFrontendGlobalNamespaceRPS is workflow namespace rate limit per second across
+	// all internal-frontends.
+	InternalFrontendGlobalNamespaceRPS = "internal-frontend.globalNamespaceRPS"
 	// FrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second for the whole cluster for visibility API.
 	// The limit is evenly distributed among available frontend service instances.
 	// If this is set, it overwrites per instance limit "frontend.namespaceRPS.visibility".
 	// This config is EXPERIMENTAL and may be changed or removed in a later release.
 	FrontendGlobalNamespaceVisibilityRPS = "frontend.globalNamespaceRPS.visibility"
+	// InternalFrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second
+	// across all internal-frontends.
+	// This config is EXPERIMENTAL and may be changed or removed in a later release.
+	InternalFrontendGlobalNamespaceVisibilityRPS = "internal-frontend.globalNamespaceRPS.visibility"
 	// FrontendThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger
 	FrontendThrottledLogRPS = "frontend.throttledLogRPS"
 	// FrontendShutdownDrainDuration is the duration of traffic drain during shutdown

service/frontend/fx.go

@@ -270,14 +270,28 @@ func RateLimitInterceptorProvider(
 }
 
 func NamespaceRateLimitInterceptorProvider(
+	serviceName primitives.ServiceName,
 	serviceConfig *Config,
 	namespaceRegistry namespace.Registry,
 	frontendServiceResolver membership.ServiceResolver,
 ) *interceptor.NamespaceRateLimitInterceptor {
+	var globalNamespaceRPS, globalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
+
+	switch serviceName {
+	case primitives.FrontendService:
+		globalNamespaceRPS = serviceConfig.GlobalNamespaceRPS
+		globalNamespaceVisibilityRPS = serviceConfig.GlobalNamespaceVisibilityRPS
+	case primitives.InternalFrontendService:
+		globalNamespaceRPS = serviceConfig.InternalFEGlobalNamespaceRPS
+		globalNamespaceVisibilityRPS = serviceConfig.InternalFEGlobalNamespaceVisibilityRPS
+	default:
+		panic("invalid service name")
+	}
+
 	rateFn := func(namespace string) float64 {
 		return namespaceRPS(
 			serviceConfig.MaxNamespaceRPSPerInstance,
-			serviceConfig.GlobalNamespaceRPS,
+			globalNamespaceRPS,
 			frontendServiceResolver,
 			namespace,
 		)
@@ -286,7 +300,7 @@ func NamespaceRateLimitInterceptorProvider(
 	visibilityRateFn := func(namespace string) float64 {
 		return namespaceRPS(
 			serviceConfig.MaxNamespaceVisibilityRPSPerInstance,
-			serviceConfig.GlobalNamespaceVisibilityRPS,
+			globalNamespaceVisibilityRPS,
 			frontendServiceResolver,
 			namespace,
 		)
@@ -393,8 +407,11 @@ func FEReplicatorNamespaceReplicationQueueProvider(
 	return replicatorNamespaceReplicationQueue
 }
 
-func ServiceResolverProvider(membershipMonitor membership.Monitor) (membership.ServiceResolver, error) {
-	return membershipMonitor.GetResolver(primitives.FrontendService)
+func ServiceResolverProvider(
+	membershipMonitor membership.Monitor,
+	serviceName primitives.ServiceName,
+) (membership.ServiceResolver, error) {
+	return membershipMonitor.GetResolver(serviceName)
 }
 
 func AdminHandlerProvider(

service/frontend/service.go

@@ -79,7 +79,9 @@ type Config struct {
 	MaxNamespaceVisibilityRPSPerInstance   dynamicconfig.IntPropertyFnWithNamespaceFilter
 	MaxNamespaceVisibilityBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
 	GlobalNamespaceRPS                     dynamicconfig.IntPropertyFnWithNamespaceFilter
+	InternalFEGlobalNamespaceRPS           dynamicconfig.IntPropertyFnWithNamespaceFilter
 	GlobalNamespaceVisibilityRPS           dynamicconfig.IntPropertyFnWithNamespaceFilter
+	InternalFEGlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
 	MaxIDLengthLimit                       dynamicconfig.IntPropertyFn
 	WorkerBuildIdSizeLimit                 dynamicconfig.IntPropertyFn
 	DisallowQuery                          dynamicconfig.BoolPropertyFnWithNamespaceFilter
@@ -193,7 +195,9 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, esIndexName
 		MaxNamespaceVisibilityRPSPerInstance:   dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance, 10),
 		MaxNamespaceVisibilityBurstPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityBurstPerInstance, 10),
 		GlobalNamespaceRPS:                     dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceRPS, 0),
+		InternalFEGlobalNamespaceRPS:           dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceRPS, 0),
 		GlobalNamespaceVisibilityRPS:           dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceVisibilityRPS, 0),
+		InternalFEGlobalNamespaceVisibilityRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceVisibilityRPS, 0),
 		MaxIDLengthLimit:                       dc.GetIntProperty(dynamicconfig.MaxIDLengthLimit, 1000),
 		WorkerBuildIdSizeLimit:                 dc.GetIntProperty(dynamicconfig.WorkerBuildIdSizeLimit, 1000),
 		MaxBadBinaries:                         dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxBadBinaries, namespace.MaxBadBinaries),