Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add separate dynamic config knobs for internal-frontend rate limiting #3800

Merged
merged 1 commit into from
Jan 12, 2023
Merged

Add separate dynamic config knobs for internal-frontend rate limiting #3800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion common/dynamicconfig/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ const (
FrontendMaxNamespaceRPSPerInstance = "frontend.namespaceRPS"
// FrontendMaxNamespaceBurstPerInstance is workflow namespace burst limit
FrontendMaxNamespaceBurstPerInstance = "frontend.namespaceBurst"
// FrontendMaxNamespaceCountPerInstance is workflow namespace count limit per second
// FrontendMaxNamespaceCountPerInstance limits concurrent task queue polls per namespace per instance
FrontendMaxNamespaceCountPerInstance = "frontend.namespaceCount"
// FrontendMaxNamespaceVisibilityRPSPerInstance is namespace rate limit per second for visibility APIs.
// This config is EXPERIMENTAL and may be changed or removed in a later release.
Expand All @@ -177,11 +177,18 @@ const (
// The limit is evenly distributed among available frontend service instances.
// If this is set, it overwrites per instance limit "frontend.namespaceRPS".
FrontendGlobalNamespaceRPS = "frontend.globalNamespaceRPS"
// InternalFrontendGlobalNamespaceRPS is workflow namespace rate limit per second across
// all internal-frontends.
InternalFrontendGlobalNamespaceRPS = "internal-frontend.globalNamespaceRPS"
// FrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second for the whole cluster for visibility API.
// The limit is evenly distributed among available frontend service instances.
// If this is set, it overwrites per instance limit "frontend.namespaceRPS.visibility".
// This config is EXPERIMENTAL and may be changed or removed in a later release.
FrontendGlobalNamespaceVisibilityRPS = "frontend.globalNamespaceRPS.visibility"
// InternalFrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second
// across all internal-frontends.
// This config is EXPERIMENTAL and may be changed or removed in a later release.
InternalFrontendGlobalNamespaceVisibilityRPS = "internal-frontend.globalNamespaceRPS.visibility"
// FrontendThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger
FrontendThrottledLogRPS = "frontend.throttledLogRPS"
// FrontendShutdownDrainDuration is the duration of traffic drain during shutdown
Expand Down
25 changes: 21 additions & 4 deletions service/frontend/fx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,14 +270,28 @@ func RateLimitInterceptorProvider(
}

func NamespaceRateLimitInterceptorProvider(
serviceName primitives.ServiceName,
serviceConfig *Config,
namespaceRegistry namespace.Registry,
frontendServiceResolver membership.ServiceResolver,
) *interceptor.NamespaceRateLimitInterceptor {
var globalNamespaceRPS, globalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter

switch serviceName {
case primitives.FrontendService:
globalNamespaceRPS = serviceConfig.GlobalNamespaceRPS
globalNamespaceVisibilityRPS = serviceConfig.GlobalNamespaceVisibilityRPS
case primitives.InternalFrontendService:
globalNamespaceRPS = serviceConfig.InternalFEGlobalNamespaceRPS
globalNamespaceVisibilityRPS = serviceConfig.InternalFEGlobalNamespaceVisibilityRPS
default:
panic("invalid service name")
}

rateFn := func(namespace string) float64 {
return namespaceRPS(
serviceConfig.MaxNamespaceRPSPerInstance,
serviceConfig.GlobalNamespaceRPS,
globalNamespaceRPS,
frontendServiceResolver,
namespace,
)
Expand All @@ -286,7 +300,7 @@ func NamespaceRateLimitInterceptorProvider(
visibilityRateFn := func(namespace string) float64 {
return namespaceRPS(
serviceConfig.MaxNamespaceVisibilityRPSPerInstance,
serviceConfig.GlobalNamespaceVisibilityRPS,
globalNamespaceVisibilityRPS,
frontendServiceResolver,
namespace,
)
Expand Down Expand Up @@ -393,8 +407,11 @@ func FEReplicatorNamespaceReplicationQueueProvider(
return replicatorNamespaceReplicationQueue
}

func ServiceResolverProvider(membershipMonitor membership.Monitor) (membership.ServiceResolver, error) {
return membershipMonitor.GetResolver(primitives.FrontendService)
func ServiceResolverProvider(
membershipMonitor membership.Monitor,
serviceName primitives.ServiceName,
) (membership.ServiceResolver, error) {
return membershipMonitor.GetResolver(serviceName)
}

func AdminHandlerProvider(
Expand Down
4 changes: 4 additions & 0 deletions service/frontend/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ type Config struct {
MaxNamespaceVisibilityRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
MaxNamespaceVisibilityBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
GlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
InternalFEGlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
GlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
InternalFEGlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
MaxIDLengthLimit dynamicconfig.IntPropertyFn
WorkerBuildIdSizeLimit dynamicconfig.IntPropertyFn
DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter
Expand Down Expand Up @@ -193,7 +195,9 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, esIndexName
MaxNamespaceVisibilityRPSPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance, 10),
MaxNamespaceVisibilityBurstPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityBurstPerInstance, 10),
GlobalNamespaceRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceRPS, 0),
InternalFEGlobalNamespaceRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceRPS, 0),
GlobalNamespaceVisibilityRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceVisibilityRPS, 0),
InternalFEGlobalNamespaceVisibilityRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.InternalFrontendGlobalNamespaceVisibilityRPS, 0),
MaxIDLengthLimit: dc.GetIntProperty(dynamicconfig.MaxIDLengthLimit, 1000),
WorkerBuildIdSizeLimit: dc.GetIntProperty(dynamicconfig.WorkerBuildIdSizeLimit, 1000),
MaxBadBinaries: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxBadBinaries, namespace.MaxBadBinaries),
Expand Down