go.temporal.io/server/common/dynamicconfig/constants.go
3595 LOC · 0 covered · 3595 uncovered · 0 ranges · 0 concepts · 0 introducers · 0 tests
1
package dynamicconfig
2
3
import (
4
"math"
5
"os"
6
"time"
7
8
sdkworker "go.temporal.io/sdk/worker"
9
"go.temporal.io/server/common/debug"
10
"go.temporal.io/server/common/primitives"
11
"go.temporal.io/server/common/retrypolicy"
12
"go.temporal.io/server/common/util"
13
"go.temporal.io/server/service/matching/counter"
14
)
15
16
var (
17
// keys for dynamic config itself
18
DynamicConfigSubscriptionPollInterval = NewGlobalDurationSetting(
19
"dynamicconfig.subscriptionPollInterval",
20
time.Minute,
21
`Poll interval for emulating subscriptions on non-subscribable Client.`,
22
)
23
24
// keys for admin
25
26
AdminEnableListHistoryTasks = NewGlobalBoolSetting(
27
"admin.enableListHistoryTasks",
28
true,
29
`AdminEnableListHistoryTasks is the key for enabling listing history tasks`,
30
)
31
AdminMatchingNamespaceToPartitionDispatchRate = NewNamespaceFloatSetting(
32
"admin.matchingNamespaceToPartitionDispatchRate",
33
10000,
34
`AdminMatchingNamespaceToPartitionDispatchRate is the max qps of any task queue partition for a given namespace`,
35
)
36
AdminMatchingNamespaceTaskqueueToPartitionDispatchRate = NewTaskQueueFloatSetting(
37
"admin.matchingNamespaceTaskqueueToPartitionDispatchRate",
38
1000,
39
`AdminMatchingNamespaceTaskqueueToPartitionDispatchRate is the max qps of a task queue partition for a given namespace & task queue`,
40
)
41
42
// keys for system
43
44
VisibilityPersistenceMaxReadQPS = NewGlobalIntSetting(
45
"system.visibilityPersistenceMaxReadQPS",
46
9000,
47
`VisibilityPersistenceMaxReadQPS is the max QPC system host can query visibility DB for read.`,
48
)
49
VisibilityPersistenceMaxWriteQPS = NewGlobalIntSetting(
50
"system.visibilityPersistenceMaxWriteQPS",
51
9000,
52
`VisibilityPersistenceMaxWriteQPS is the max QPC system host can query visibility DB for write.`,
53
)
54
VisibilityPersistenceSlowQueryThreshold = NewGlobalDurationSetting(
55
"system.visibilityPersistenceSlowQueryThreshold",
56
time.Second,
57
`VisibilityPersistenceSlowQueryThreshold is the threshold above which a query is considered slow and logged.`,
58
)
59
EnableReadFromSecondaryVisibility = NewNamespaceBoolSetting(
60
"system.enableReadFromSecondaryVisibility",
61
false,
62
`EnableReadFromSecondaryVisibility is the config to enable read from secondary visibility`,
63
)
64
VisibilityEnableShadowReadMode = NewGlobalBoolSetting(
65
"system.visibilityEnableShadowReadMode",
66
false,
67
`VisibilityEnableShadowReadMode is the config to enable shadow read from secondary visibility`,
68
)
69
SecondaryVisibilityWritingMode = NewGlobalStringSetting(
70
"system.secondaryVisibilityWritingMode",
71
"off",
72
`SecondaryVisibilityWritingMode is key for how to write to secondary visibility`,
73
)
74
VisibilityDisableOrderByClause = NewNamespaceBoolSetting(
75
"system.visibilityDisableOrderByClause",
76
true,
77
`VisibilityDisableOrderByClause is the config to disable ORDERY BY clause for Elasticsearch`,
78
)
79
VisibilityEnableManualPagination = NewNamespaceBoolSetting(
80
"system.visibilityEnableManualPagination",
81
true,
82
`VisibilityEnableManualPagination is the config to enable manual pagination for Elasticsearch`,
83
)
84
VisibilityAllowList = NewNamespaceBoolSetting(
85
"system.visibilityAllowList",
86
true,
87
`VisibilityAllowList is the config to allow list of values for regular types`,
88
)
89
SuppressErrorSetSystemSearchAttribute = NewNamespaceBoolSetting(
90
"system.suppressErrorSetSystemSearchAttribute",
91
false,
92
`SuppressErrorSetSystemSearchAttribute suppresses errors when trying to set
93
values in system search attributes.`,
94
)
95
VisibilityEnableUnifiedQueryConverter = NewGlobalBoolSetting(
96
"system.visibilityEnableUnifiedQueryConverter",
97
false,
98
`VisibilityEnableUnifiedQueryConverter enables the unified query converter for parsing the
99
query.`,
100
)
101
102
HistoryArchivalState = NewGlobalStringSetting(
103
"system.historyArchivalState",
104
"", // actual default is from static config
105
`HistoryArchivalState is key for the state of history archival`,
106
)
107
EnableReadFromHistoryArchival = NewGlobalBoolSetting(
108
"system.enableReadFromHistoryArchival",
109
false, // actual default is from static config
110
`EnableReadFromHistoryArchival is key for enabling reading history from archival store`,
111
)
112
VisibilityArchivalState = NewGlobalStringSetting(
113
"system.visibilityArchivalState",
114
"", // actual default is from static config
115
`VisibilityArchivalState is key for the state of visibility archival`,
116
)
117
EnableReadFromVisibilityArchival = NewGlobalBoolSetting(
118
"system.enableReadFromVisibilityArchival",
119
false, // actual default is from static config
120
`EnableReadFromVisibilityArchival is key for enabling reading visibility from archival store`,
121
)
122
EnableNamespaceNotActiveAutoForwarding = NewNamespaceBoolSetting(
123
"system.enableNamespaceNotActiveAutoForwarding",
124
true,
125
`EnableNamespaceNotActiveAutoForwarding whether enabling DC auto forwarding to active cluster
126
for signal / start / signal with start API if namespace is not active`,
127
)
128
ForceNamespaceSelectedAPIAutoForwarding = NewNamespaceBoolSetting(
129
"system.forceNamespaceSelectedAPIAutoForwarding",
130
false,
131
`ForceNamespaceSelectedAPIAutoForwarding forces selective (whitelist) API forwarding for the namespace when true, overriding all-apis-forwarding policy for that namespace`,
132
)
133
EnableNamespaceHandoverWait = NewNamespaceBoolSetting(
134
"system.enableNamespaceHandoverWait",
135
false,
136
`EnableNamespaceHandoverWait whether waiting for namespace replication state update before serve the request`,
137
)
138
TransactionSizeLimit = NewGlobalIntSetting(
139
"system.transactionSizeLimit",
140
primitives.DefaultTransactionSizeLimit,
141
`TransactionSizeLimit is the largest allowed transaction size to persistence`,
142
)
143
DisallowQuery = NewNamespaceBoolSetting(
144
"system.disallowQuery",
145
false,
146
`DisallowQuery is the key to disallow query for a namespace`,
147
)
148
EnableCrossNamespaceCommands = NewGlobalBoolSetting(
149
"system.enableCrossNamespaceCommands",
150
false,
151
`EnableCrossNamespaceCommands is the key to enable commands for external namespaces`,
152
)
153
DisableStreamingAuthorizer = NewGlobalBoolSetting(
154
"system.disableStreamingAuthorizer",
155
false,
156
`DisableStreamingAuthorizer is the key to disable the auth on streaming endpoint`,
157
)
158
RetryUnboundedOnSystemResourceExhausted = NewGlobalBoolSetting(
159
"system.retryUnboundedOnSystemResourceExhausted",
160
false,
161
`RetryUnboundedOnSystemResourceExhausted controls retry behavior of inter-service
162
calls (frontend, CHASM, etc.) to history and matching on system-scoped ResourceExhausted
163
errors. When false (the default), these calls follow the standard 2-attempt cap. When
164
true, they ignore the attempt cap and keep retrying for up to the policy's expiration
165
interval (1 minute) or until the caller's context is cancelled, whichever comes first.`,
166
)
167
ClusterMetadataRefreshInterval = NewGlobalDurationSetting(
168
"system.clusterMetadataRefreshInterval",
169
time.Minute,
170
`ClusterMetadataRefreshInterval is config to manage cluster metadata table refresh interval`,
171
)
172
ForceSearchAttributesCacheRefreshOnRead = NewGlobalBoolSetting(
173
"system.forceSearchAttributesCacheRefreshOnRead",
174
false,
175
`ForceSearchAttributesCacheRefreshOnRead forces refreshing search attributes cache on a read operation, so we always
176
get the latest data from DB. This effectively bypasses cache value and is used to facilitate testing of changes in
177
search attributes. This should not be turned on in production.`,
178
)
179
EnableRingpopTLS = NewGlobalBoolSetting(
180
"system.enableRingpopTLS",
181
false,
182
`EnableRingpopTLS controls whether to use TLS for ringpop, using the same "internode" TLS
183
config as the other services.`,
184
)
185
RingpopApproximateMaxPropagationTime = NewGlobalDurationSetting(
186
"system.ringpopApproximateMaxPropagationTime",
187
3*time.Second,
188
`RingpopApproximateMaxPropagationTime is used for timing certain startup and shutdown processes.
189
(It is not and doesn't have to be a guarantee.)`,
190
)
191
RingpopReplicaPoints = NewGlobalIntSetting(
192
"system.ringpopReplicaPoints",
193
100,
194
`RingpopReplicaPoints is the number of virtual nodes (replica points) per physical host
195
in the consistent hash ring used by ringpop. Changing it may cause service disruption during deployment.`,
196
)
197
EnableParentClosePolicyWorker = NewGlobalBoolSetting(
198
"system.enableParentClosePolicyWorker",
199
true,
200
`EnableParentClosePolicyWorker decides whether or not enable system workers for processing parent close policy task`,
201
)
202
EnableStickyQuery = NewNamespaceBoolSetting(
203
"system.enableStickyQuery",
204
true,
205
`EnableStickyQuery indicates if sticky query should be enabled per namespace`,
206
)
207
EnableActivityEagerExecution = NewNamespaceBoolSetting(
208
"system.enableActivityEagerExecution",
209
true,
210
`EnableActivityEagerExecution indicates if activity eager execution is enabled per namespace`,
211
)
212
EnableCancelActivityWorkerCommand = NewNamespaceBoolSetting(
213
"system.enableCancelActivityWorkerCommand",
214
false,
215
`EnableCancelActivityWorkerCommand enables pushing activity cancellation to workers via Nexus worker commands`,
216
)
217
NamespaceMinRetentionGlobal = NewGlobalDurationSetting(
218
"system.namespaceMinRetentionGlobal",
219
24*time.Hour,
220
`Minimum retention duration for global namespaces. This value should only be lowered for testing purposes.`,
221
)
222
NamespaceMinRetentionLocal = NewGlobalDurationSetting(
223
"system.namespaceMinRetentionLocal",
224
time.Hour,
225
`Minimum retention duration for local namespaces. This value should only be lowered for testing purposes.`,
226
)
227
EnableActivityRetryStampIncrement = NewGlobalBoolSetting(
228
"system.enableActivityRetryStampIncrement",
229
false,
230
`EnableActivityRetryStampIncrement indicates if activity retry stamp increment is enabled`,
231
)
232
EnableEagerWorkflowStart = NewNamespaceBoolSetting(
233
"system.enableEagerWorkflowStart",
234
true,
235
`Toggles "eager workflow start" - returning the first workflow task inline in the
236
response to a StartWorkflowExecution request and skipping the trip through matching.`,
237
)
238
NamespaceCacheRefreshInterval = NewGlobalDurationSetting(
239
"system.namespaceCacheRefreshInterval",
240
2*time.Second,
241
`NamespaceCacheRefreshInterval is the key for namespace cache refresh interval dynamic config`,
242
)
243
PersistenceHealthSignalMetricsEnabled = NewGlobalBoolSetting(
244
"system.persistenceHealthSignalMetricsEnabled",
245
true,
246
`PersistenceHealthSignalMetricsEnabled determines whether persistence shard RPS metrics are emitted`,
247
)
248
HistoryHealthSignalMetricsEnabled = NewGlobalBoolSetting(
249
"system.historyHealthSignalMetricsEnabled",
250
true,
251
`HistoryHealthSignalMetricsEnabled determines whether history service RPC metrics are emitted`,
252
)
253
HistoryHealthSignalLatencyWindowCount = NewGlobalIntSetting(
254
"system.historyHealthSignalLatencyWindowCount",
255
10,
256
`historyHealthSignalLatencyWindowCount is the number of signal windows to compute latencies over`,
257
)
258
HistoryHealthSignalLatencyWindowSize = NewGlobalDurationSetting(
259
"system.historyHealthSignalLatencyWindowSize",
260
5*time.Second,
261
`historyHealthSignalLatencyWindowSize is the time window size in seconds for aggregating latencies`,
262
)
263
HistoryHealthSignalPercentileLatencySettings = NewGlobalTypedSetting(
264
"system.historyHealthSignalPercentileLatencySettings",
265
LatencyHealthChecksPerPercentile{},
266
"historyHealthSignalPercentileLatencySettings controls what latency health checks are enabled and enforced for the history system",
267
)
268
// TODO: This should be removed once percentiles are the default.
269
HistoryHealthSignalUsePercentiles = NewGlobalBoolSetting(
270
"system.historyHealthSignalUsePercentiles",
271
false,
272
`historyHealthSignalUsePercentiles controls whether we use the p99 latency for health checking instead of the mean latency`,
273
)
274
PersistenceHealthSignalAggregationEnabled = NewGlobalBoolSetting(
275
"system.persistenceHealthSignalAggregationEnabled",
276
true,
277
`PersistenceHealthSignalAggregationEnabled determines whether persistence latency and error averages are tracked`,
278
)
279
PersistenceHealthSignalPercentilesEnabled = NewGlobalBoolSetting(
280
"system.persistenceHealthSignalPercentilesEnabled",
281
false,
282
`PersistenceHealthSignalPercentilesEnabled determines whether persistence latency is tracked using distribution objects`,
283
)
284
PersistenceHealthSignalLatencyWindowCount = NewGlobalIntSetting(
285
"system.persistenceHealthSignalLatencyWindowCount",
286
10,
287
`PersistenceHealthSignalLatencyWindowCount is the number of signal windows to compute latencies over`,
288
)
289
PersistenceHealthSignalLatencyWindowSize = NewGlobalDurationSetting(
290
"system.persistenceHealthSignalLatencyWindowSize",
291
5*time.Second,
292
`PersistenceHealthSignalLatencyWindowSize is the time window size in seconds for aggregating latencies`,
293
)
294
PersistenceHealthSignalPercentileLatencySettings = NewGlobalTypedSetting(
295
"system.persistenceHealthSignalPercentileLatencySettings",
296
LatencyHealthChecksPerPercentile{},
297
"persistenceHealthSignalPercentileLatencySettings controls what latency health checks are enabled and enforced for the persistence system",
298
)
299
PersistenceHealthSignalWindowSize = NewGlobalDurationSetting(
300
"system.persistenceHealthSignalWindowSize",
301
10*time.Second,
302
`PersistenceHealthSignalWindowSize is the time window size in seconds for aggregating persistence signals`,
303
)
304
PersistenceHealthSignalBufferSize = NewGlobalIntSetting(
305
"system.persistenceHealthSignalBufferSize",
306
5000,
307
`PersistenceHealthSignalBufferSize is the maximum number of persistence signals to buffer in memory per signal key`,
308
)
309
OperatorRPSRatio = NewGlobalFloatSetting(
310
"system.operatorRPSRatio",
311
0.2,
312
`OperatorRPSRatio is the percentage of the rate limit provided to priority rate limiters that should be used for
313
operator API calls (highest priority). Should be >0.0 and <= 1.0 (defaults to 20% if not specified)`,
314
)
315
// TODO: The following 2 configs should be removed once server keepalive and client keepalive are enabled by default
316
EnableInternodeServerKeepAlive = NewGlobalBoolSetting(
317
"system.enableInternodeServerKeepAlive",
318
false,
319
`enableInternodeServerKeepAlive is the config to enable keep alive for inter-node connections on server side.`,
320
)
321
EnableInternodeClientKeepAlive = NewGlobalBoolSetting(
322
"system.enableInternodeClientKeepAlive",
323
false,
324
`enableInternodeClientKeepAlive is the config to enable keep alive for inter-node connections on client side.`,
325
)
326
327
PersistenceQPSBurstRatio = NewGlobalFloatSetting(
328
"system.persistenceQPSBurstRatio",
329
1.0,
330
`PersistenceQPSBurstRatio is the burst ratio for persistence QPS. This flag controls the burst ratio for all services.`,
331
)
332
333
EnableDataLossMetrics = NewGlobalBoolSetting(
334
"system.enableDataLossMetrics",
335
false,
336
`EnableDataLossMetrics determines whether dataloss metrics are emitted when dataloss errors are encountered`,
337
)
338
339
// deadlock detector
340
341
DeadlockDumpGoroutines = NewGlobalBoolSetting(
342
"system.deadlock.DumpGoroutines",
343
true,
344
`Whether the deadlock detector should dump goroutines`,
345
)
346
DeadlockFailHealthCheck = NewGlobalBoolSetting(
347
"system.deadlock.FailHealthCheck",
348
false,
349
`Whether the deadlock detector should cause the grpc server to fail health checks`,
350
)
351
DeadlockAbortProcess = NewGlobalBoolSetting(
352
"system.deadlock.AbortProcess",
353
false,
354
`Whether the deadlock detector should abort the process`,
355
)
356
DeadlockInterval = NewGlobalDurationSetting(
357
"system.deadlock.Interval",
358
60*time.Second,
359
`How often the detector checks each root.`,
360
)
361
DeadlockMaxWorkersPerRoot = NewGlobalIntSetting(
362
"system.deadlock.MaxWorkersPerRoot",
363
10,
364
`How many extra goroutines can be created per root.`,
365
)
366
367
NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute = NewNamespaceIntSetting(
368
"system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute",
369
5,
370
`NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute is the number of consecutive workflow task problems to trigger the TemporalReportedProblems search attribute.
371
Setting this to 0 prevents the search attribute from being set when a problem is detected, and unset when the problem is resolved.`,
372
)
373
374
PollWaitForNamespaceRateLimitToken = NewNamespaceBoolSetting(
375
"system.pollWaitForNamespaceRateLimitToken",
376
false,
377
`PollWaitForNamespaceRateLimitToken controls whether poll requests wait for
378
a namespace RPS rate limit token to become available instead of immediately rejecting
379
with ResourceExhausted. When enabled, poll requests block until a token is available
380
or the request context deadline is reached. The concurrent request rate limiter fires
381
before this limiter and will still reject requests that exceed the concurrent limit.`,
382
)
383
384
// keys for size limit
385
386
BlobSizeLimitError = NewNamespaceIntSetting(
387
"limit.blobSize.error",
388
2*1024*1024,
389
`BlobSizeLimitError is the per event blob size limit`,
390
)
391
BlobSizeLimitWarn = NewNamespaceIntSetting(
392
"limit.blobSize.warn",
393
512*1024,
394
`BlobSizeLimitWarn is the per event blob size limit for warning`,
395
)
396
MemoSizeLimitError = NewNamespaceIntSetting(
397
"limit.memoSize.error",
398
2*1024*1024,
399
`MemoSizeLimitError is the per event memo size limit`,
400
)
401
MemoSizeLimitWarn = NewNamespaceIntSetting(
402
"limit.memoSize.warn",
403
2*1024,
404
`MemoSizeLimitWarn is the per event memo size limit for warning`,
405
)
406
NumPendingChildExecutionsLimitError = NewNamespaceIntSetting(
407
"limit.numPendingChildExecutions.error",
408
2000,
409
`NumPendingChildExecutionsLimitError is the maximum number of pending child workflows a workflow can have before
410
StartChildWorkflowExecution commands will fail.`,
411
)
412
NumPendingActivitiesLimitError = NewNamespaceIntSetting(
413
"limit.numPendingActivities.error",
414
2000,
415
`NumPendingActivitiesLimitError is the maximum number of pending activities a workflow can have before
416
ScheduleActivityTask will fail.`,
417
)
418
NumPendingSignalsLimitError = NewNamespaceIntSetting(
419
"limit.numPendingSignals.error",
420
2000,
421
`NumPendingSignalsLimitError is the maximum number of pending signals a workflow can have before
422
SignalExternalWorkflowExecution commands from this workflow will fail.`,
423
)
424
NumPendingCancelRequestsLimitError = NewNamespaceIntSetting(
425
"limit.numPendingCancelRequests.error",
426
2000,
427
`NumPendingCancelRequestsLimitError is the maximum number of pending requests to cancel other workflows a workflow can have before
428
RequestCancelExternalWorkflowExecution commands will fail.`,
429
)
430
HistorySizeLimitError = NewNamespaceIntSetting(
431
"limit.historySize.error",
432
50*1024*1024,
433
`HistorySizeLimitError is the per workflow execution history size limit`,
434
)
435
HistorySizeLimitWarn = NewNamespaceIntSetting(
436
"limit.historySize.warn",
437
10*1024*1024,
438
`HistorySizeLimitWarn is the per workflow execution history size limit for warning`,
439
)
440
HistorySizeSuggestContinueAsNew = NewNamespaceIntSetting(
441
"limit.historySize.suggestContinueAsNew",
442
4*1024*1024,
443
`HistorySizeSuggestContinueAsNew is the workflow execution history size limit to suggest
444
continue-as-new (in workflow task started event)`,
445
)
446
HistoryCountLimitError = NewNamespaceIntSetting(
447
"limit.historyCount.error",
448
50*1024,
449
`HistoryCountLimitError is the per workflow execution history event count limit`,
450
)
451
HistoryCountLimitWarn = NewNamespaceIntSetting(
452
"limit.historyCount.warn",
453
10*1024,
454
`HistoryCountLimitWarn is the per workflow execution history event count limit for warning`,
455
)
456
MutableStateActivityFailureSizeLimitError = NewNamespaceIntSetting(
457
"limit.mutableStateActivityFailureSize.error",
458
4*1024,
459
`MutableStateActivityFailureSizeLimitError is the per activity failure size limit for workflow mutable state.
460
If exceeded, failure will be truncated before being stored in mutable state.`,
461
)
462
MutableStateActivityFailureSizeLimitWarn = NewNamespaceIntSetting(
463
"limit.mutableStateActivityFailureSize.warn",
464
2*1024,
465
`MutableStateActivityFailureSizeLimitWarn is the per activity failure size warning limit for workflow mutable state`,
466
)
467
MutableStateSizeLimitError = NewGlobalIntSetting(
468
"limit.mutableStateSize.error",
469
8*1024*1024,
470
`MutableStateSizeLimitError is the per workflow execution mutable state size limit in bytes`,
471
)
472
MutableStateSizeLimitWarn = NewGlobalIntSetting(
473
"limit.mutableStateSize.warn",
474
1*1024*1024,
475
`MutableStateSizeLimitWarn is the per workflow execution mutable state size limit in bytes for warning`,
476
)
477
MutableStateTombstoneCountLimit = NewGlobalIntSetting(
478
"limit.mutableStateTombstoneCountLimit",
479
16,
480
`MutableStateTombstoneCountLimit is the maximum number of deleted sub state machines tracked in mutable state.`,
481
)
482
HistoryCountSuggestContinueAsNew = NewNamespaceIntSetting(
483
"limit.historyCount.suggestContinueAsNew",
484
4*1024,
485
`HistoryCountSuggestContinueAsNew is the workflow execution history event count limit to
486
suggest continue-as-new (in workflow task started event)`,
487
)
488
HistoryMaxPageSize = NewNamespaceIntSetting(
489
"limit.historyMaxPageSize",
490
primitives.GetHistoryMaxPageSize,
491
`HistoryMaxPageSize is default max size for GetWorkflowExecutionHistory in one page`,
492
)
493
MaxIDLengthLimit = NewGlobalIntSetting(
494
"limit.maxIDLength",
495
1000,
496
`MaxIDLengthLimit is the length limit for various IDs, including: Namespace, TaskQueue, WorkflowID, ActivityID, TimerID,
497
WorkflowType, ActivityType, SignalName, MarkerName, ErrorReason/FailureReason/CancelCause, Identity, RequestID`,
498
)
499
WorkerBuildIdSizeLimit = NewGlobalIntSetting(
500
"limit.workerBuildIdSize",
501
255,
502
`WorkerBuildIdSizeLimit is the byte length limit for a worker build id as used in the rpc methods for updating
503
the version sets for a task queue.
504
Do not set this to a value higher than 255 for clusters using SQL based persistence due to predefined VARCHAR
505
column width.`,
506
)
507
VersionCompatibleSetLimitPerQueue = NewNamespaceIntSetting(
508
"limit.versionCompatibleSetLimitPerQueue",
509
10,
510
`VersionCompatibleSetLimitPerQueue is the max number of compatible sets allowed in the versioning data for a task
511
queue. Update requests which would cause the versioning data to exceed this number will fail with a
512
FailedPrecondition error.`,
513
)
514
VersionBuildIdLimitPerQueue = NewNamespaceIntSetting(
515
"limit.versionBuildIdLimitPerQueue",
516
100,
517
`VersionBuildIdLimitPerQueue is the max number of build IDs allowed to be defined in the versioning data for a
518
task queue. Update requests which would cause the versioning data to exceed this number will fail with a
519
FailedPrecondition error.`,
520
)
521
AssignmentRuleLimitPerQueue = NewNamespaceIntSetting(
522
"limit.wv.AssignmentRuleLimitPerQueue",
523
100,
524
`AssignmentRuleLimitPerQueue is the max number of Build ID assignment rules allowed to be defined in the
525
versioning data for a task queue. Update requests which would cause the versioning data to exceed this number
526
will fail with a FailedPrecondition error.`,
527
)
528
RedirectRuleLimitPerQueue = NewNamespaceIntSetting(
529
"limit.wv.RedirectRuleLimitPerQueue",
530
500,
531
`RedirectRuleLimitPerQueue is the max number of compatible redirect rules allowed to be defined
532
in the versioning data for a task queue. Update requests which would cause the versioning data to exceed this
533
number will fail with a FailedPrecondition error.`,
534
)
535
RedirectRuleMaxUpstreamBuildIDsPerQueue = NewNamespaceIntSetting(
536
"limit.wv.RedirectRuleMaxUpstreamBuildIDsPerQueue",
537
50,
538
`RedirectRuleMaxUpstreamBuildIDsPerQueue is the max number of compatible redirect rules allowed to be connected
539
in one chain in the versioning data for a task queue. Update requests which would cause the versioning data
540
to exceed this number will fail with a FailedPrecondition error.`,
541
)
542
MatchingDeletedRuleRetentionTime = NewNamespaceDurationSetting(
543
"matching.wv.DeletedRuleRetentionTime",
544
14*24*time.Hour,
545
`MatchingDeletedRuleRetentionTime is the length of time that deleted Version Assignment Rules and
546
Deleted Redirect Rules will be kept in the DB (with DeleteTimestamp). After this time, the tombstones are deleted at the next time update of versioning data for the task queue.`,
547
)
548
PollerHistoryTTL = NewNamespaceDurationSetting(
549
"matching.PollerHistoryTTL",
550
5*time.Minute,
551
`PollerHistoryTTL is the time to live for poller histories in the pollerHistory cache of a physical task queue. Poller histories are fetched when
552
requiring a list of pollers that polled a given task queue.`,
553
)
554
ReachabilityBuildIdVisibilityGracePeriod = NewNamespaceDurationSetting(
555
"matching.wv.ReachabilityBuildIdVisibilityGracePeriod",
556
3*time.Minute,
557
`ReachabilityBuildIdVisibilityGracePeriod is the time period for which deleted versioning rules are still considered active
558
to account for the delay in updating the build id field in visibility. Not yet supported for GetDeploymentReachability. We recommend waiting
559
at least 2 minutes between changing the current deployment and calling GetDeployment, so that newly started workflow executions using the
560
recently-current deployment can arrive in visibility.`,
561
)
562
VersionDrainageStatusVisibilityGracePeriod = NewNamespaceDurationSetting(
563
"matching.wv.VersionDrainageStatusVisibilityGracePeriod",
564
3*time.Minute,
565
`VersionDrainageStatusVisibilityGracePeriod is the time period for which non-current / non-ramping worker deployment versions
566
are still considered active to account for the delay in updating the build id field in visibility.`,
567
)
568
VersionDrainageStatusRefreshInterval = NewNamespaceDurationSetting(
569
"matching.wv.VersionDrainageStatusRefreshInterval",
570
3*time.Minute,
571
`VersionDrainageStatusRefreshInterval is the interval at which each draining deployment version refreshes its
572
Drainage Status by querying visibility for open pinned workflows using that version.`,
573
)
574
ReachabilityTaskQueueScanLimit = NewGlobalIntSetting(
575
"limit.reachabilityTaskQueueScan",
576
20,
577
`ReachabilityTaskQueueScanLimit limits the number of task queues to scan when responding to a
578
GetWorkerTaskReachability query.`,
579
)
580
ReachabilityQueryBuildIdLimit = NewGlobalIntSetting(
581
"limit.reachabilityQueryBuildIds",
582
5,
583
`ReachabilityQueryBuildIdLimit limits the number of build ids that can be requested in a single call to the
584
DescribeTaskQueue API with ReportTaskQueueReachability==true, or to the GetWorkerTaskReachability API.`,
585
)
586
ReachabilityCacheOpenWFsTTL = NewGlobalDurationSetting(
587
"matching.wv.reachabilityCacheOpenWFsTTL",
588
time.Minute,
589
`ReachabilityCacheOpenWFsTTL is the TTL for the reachability open workflows cache.`,
590
)
591
ReachabilityCacheClosedWFsTTL = NewGlobalDurationSetting(
592
"matching.wv.reachabilityCacheClosedWFsTTL",
593
10*time.Minute,
594
`ReachabilityCacheClosedWFsTTL is the TTL for the reachability closed workflows cache.`,
595
)
596
ReachabilityQuerySetDurationSinceDefault = NewGlobalDurationSetting(
597
"frontend.reachabilityQuerySetDurationSinceDefault",
598
5*time.Minute,
599
`ReachabilityQuerySetDurationSinceDefault is the minimum period since a version set was demoted from being the
600
queue default before it is considered unreachable by new workflows.
601
This setting allows some propagation delay of versioning data for the reachability queries, which may happen for
602
the following reasons:
603
1. There are no workflows currently marked as open in the visibility store but a worker for the demoted version
604
is currently processing a task.
605
2. There are delays in the visibility task processor (which is asynchronous).
606
3. There's propagation delay of the versioning data between matching nodes.`,
607
)
608
TaskQueuesPerBuildIdLimit = NewNamespaceIntSetting(
609
"limit.taskQueuesPerBuildId",
610
20,
611
`TaskQueuesPerBuildIdLimit limits the number of task queue names that can be mapped to a single build id.`,
612
)
613
614
NexusEndpointNameMaxLength = NewGlobalIntSetting(
615
"limit.endpointNameMaxLength",
616
200,
617
`NexusEndpointNameMaxLength is the maximum length of a Nexus endpoint name.`,
618
)
619
NexusEndpointExternalURLMaxLength = NewGlobalIntSetting(
620
"limit.endpointExternalURLMaxLength",
621
4*1024,
622
`NexusEndpointExternalURLMaxLength is the maximum length of a Nexus endpoint external target URL.`,
623
)
624
NexusEndpointDescriptionMaxSize = NewNamespaceIntSetting(
625
"limit.endpointDescriptionMaxSize",
626
20000,
627
`Maximum size of Nexus Endpoint description payload in bytes including data and metadata.`,
628
)
629
NexusEndpointListDefaultPageSize = NewGlobalIntSetting(
630
"limit.endpointListDefaultPageSize",
631
100,
632
`NexusEndpointListDefaultPageSize is the default page size for listing Nexus endpoints.`,
633
)
634
NexusEndpointListMaxPageSize = NewGlobalIntSetting(
635
"limit.endpointListMaxPageSize",
636
1000,
637
`NexusEndpointListMaxPageSize is the maximum page size for listing Nexus endpoints.`,
638
)
639
640
RemovableBuildIdDurationSinceDefault = NewGlobalDurationSetting(
641
"worker.removableBuildIdDurationSinceDefault",
642
time.Hour,
643
`RemovableBuildIdDurationSinceDefault is the minimum duration since a build id was last default in its containing
644
set for it to be considered for removal, used by the build id scavenger.
645
This setting allows some propagation delay of versioning data, which may happen for the following reasons:
646
1. There are no workflows currently marked as open in the visibility store but a worker for the demoted version
647
is currently processing a task.
648
2. There are delays in the visibility task processor (which is asynchronous).
649
3. There's propagation delay of the versioning data between matching nodes.`,
650
)
651
BuildIdScavengerVisibilityRPS = NewGlobalFloatSetting(
652
"worker.buildIdScavengerVisibilityRPS",
653
1.0,
654
`BuildIdScavengerVisibilityRPS is the rate limit for visibility calls from the build id scavenger`,
655
)
656
657
ScheduleInvariantsScannerOptions = NewGlobalTypedSetting(
658
"worker.scheduleInvariantsScannerOptions",
659
DefaultScheduleInvariantsScannerParams,
660
`ScheduleInvariantsScannerOptions configures the schedule-invariants scanners.
661
Fields: OverdueNextActionTimeEnabled, StuckOpenEnabled, UnknownStateEnabled (per-invariant
662
toggles, all default false), OverdueNextActionTimeTolerance, OverdueNextActionTimeMaxChecksPerNamespace,
663
VisibilityRPS, ScanInterval, and StuckOpenIdleTimeBufferMultiplier. See
664
ScheduleInvariantsScannerParams comments for details.`,
665
)
666
667
// keys for frontend
668
FrontendAllowedExperiments = NewNamespaceTypedSetting(
669
"frontend.allowedExperiments",
670
[]string(nil),
671
`FrontendAllowedExperiments is a list of experiment names that can be enabled via the temporal-experiment header for a specific namespace.`,
672
)
673
FrontendHTTPAllowedHosts = NewGlobalTypedSettingWithConverter(
674
"frontend.httpAllowedHosts",
675
ConvertWildcardStringListToRegexp,
676
MatchAnythingRE,
677
`HTTP API Requests with a "Host" header matching the allowed hosts will be processed, otherwise rejected.
678
Wildcards (*) are expanded to allow any substring. By default any Host header is allowed.
679
Concrete type should be list of strings.`,
680
)
681
FrontendPersistenceMaxQPS = NewGlobalIntSetting(
682
"frontend.persistenceMaxQPS",
683
2000,
684
`FrontendPersistenceMaxQPS is the max qps frontend host can query DB`,
685
)
686
FrontendPersistenceGlobalMaxQPS = NewGlobalIntSetting(
687
"frontend.persistenceGlobalMaxQPS",
688
0,
689
`FrontendPersistenceGlobalMaxQPS is the max qps frontend cluster can query DB`,
690
)
691
FrontendPersistenceNamespaceMaxQPS = NewNamespaceIntSetting(
692
"frontend.persistenceNamespaceMaxQPS",
693
0,
694
`FrontendPersistenceNamespaceMaxQPS is the max qps each namespace on frontend host can query DB`,
695
)
696
FrontendPersistenceGlobalNamespaceMaxQPS = NewNamespaceIntSetting(
697
"frontend.persistenceGlobalNamespaceMaxQPS",
698
0,
699
`FrontendPersistenceGlobalNamespaceMaxQPS is the max qps each namespace in frontend cluster can query DB`,
700
)
701
FrontendPersistenceDynamicRateLimitingParams = NewGlobalTypedSetting(
702
"frontend.persistenceDynamicRateLimitingParams",
703
DefaultDynamicRateLimitingParams,
704
`FrontendPersistenceDynamicRateLimitingParams is a struct that contains all adjustable dynamic rate limiting params.
705
Fields: Enabled, RefreshInterval, LatencyThreshold, ErrorThreshold, RateBackoffStepSize, RateIncreaseStepSize, RateMultiMin, RateMultiMax.
706
See DynamicRateLimitingParams comments for more details.`,
707
)
708
FrontendVisibilityMaxPageSize = NewNamespaceIntSetting(
709
"frontend.visibilityMaxPageSize",
710
1000,
711
`FrontendVisibilityMaxPageSize is default max size for ListWorkflowExecutions in one page`,
712
)
713
FrontendHistoryMaxPageSize = NewNamespaceIntSetting(
714
"frontend.historyMaxPageSize",
715
primitives.GetHistoryMaxPageSize,
716
`FrontendHistoryMaxPageSize is default max size for GetWorkflowExecutionHistory in one page`,
717
)
718
FrontendRPS = NewGlobalIntSetting(
719
"frontend.rps",
720
2400,
721
`FrontendRPS is workflow rate limit per second per-instance`,
722
)
723
FrontendGlobalRPS = NewGlobalIntSetting(
724
"frontend.globalRPS",
725
0,
726
`FrontendGlobalRPS is workflow rate limit per second for the whole cluster`,
727
)
728
FrontendNamespaceReplicationInducingAPIsRPS = NewGlobalIntSetting(
729
"frontend.rps.namespaceReplicationInducingAPIs",
730
20,
731
`FrontendNamespaceReplicationInducingAPIsRPS limits the per second request rate for namespace replication inducing
732
APIs (e.g. RegisterNamespace, UpdateNamespace, UpdateWorkerBuildIdCompatibility).
733
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
734
)
735
FrontendMaxNamespaceRPSPerInstance = NewNamespaceIntSetting(
736
"frontend.namespaceRPS",
737
2400,
738
`FrontendMaxNamespaceRPSPerInstance is workflow namespace rate limit per second`,
739
)
740
FrontendMaxNamespaceBurstRatioPerInstance = NewNamespaceFloatSetting(
741
"frontend.namespaceBurstRatio",
742
2,
743
`FrontendMaxNamespaceBurstRatioPerInstance is workflow namespace burst limit as a ratio of namespace RPS. The RPS
744
used here will be the effective RPS from global and per-instance limits. The value must be 1 or higher.`,
745
)
746
FrontendGlobalWorkerDeploymentReadRPS = NewNamespaceIntSetting(
747
"frontend.globalNamespaceWorkerDeploymentReadRPS",
748
50,
749
`FrontendGlobalWorkerDeploymentReadRPS is the global, per-namespace rate limit for Worker Deployment Read APIs (DescribeWorkerDeployment, DescribeWorkerDeploymentVersion). The limit is evenly distributed among available frontend service instances.`,
750
)
751
FrontendGlobalWorkerDeploymentReadBurstRatio = NewNamespaceFloatSetting(
752
"frontend.globalNamespaceWorkerDeploymentReadBurstRatio",
753
10,
754
`FrontendGlobalWorkerDeploymentReadBurstRatio is the burst limit for Worker Deployment Read APIs (DescribeWorkerDeployment, DescribeWorkerDeploymentVersion) as a ratio of FrontendGlobalWorkerDeploymentReadRPS. The RPS used here is the effective per-instance RPS after distributing the global limit among available frontend service instances. The value must be 1 or higher.`,
755
)
756
FrontendMaxConcurrentLongRunningRequestsPerInstance = NewNamespaceIntSetting(
757
"frontend.namespaceCount",
758
1200,
759
`FrontendMaxConcurrentLongRunningRequestsPerInstance limits concurrent long-running requests per-instance,
760
per-API. Example requests include long-poll requests, and 'Query' requests (which need to wait for WFTs). The
761
limit is applied individually to each API method. This value is ignored if
762
FrontendGlobalMaxConcurrentLongRunningRequests is greater than zero. Warning: setting this to zero will cause all
763
long-running requests to fail. The name 'frontend.namespaceCount' is kept for backwards compatibility with
764
existing deployments even though it is a bit of a misnomer. This does not limit the number of namespaces; it is a
765
per-_namespace_ limit on the _count_ of long-running requests. Requests are only throttled when the limit is
766
exceeded, not when it is only reached.`,
767
)
768
FrontendGlobalMaxConcurrentLongRunningRequests = NewNamespaceIntSetting(
769
"frontend.globalNamespaceCount",
770
0,
771
`FrontendGlobalMaxConcurrentLongRunningRequests limits concurrent long-running requests across all frontend
772
instances in the cluster, for a given namespace, per-API method. If this is set to 0 (the default), then it is
773
ignored. The name 'frontend.globalNamespaceCount' is kept for consistency with the per-instance limit name,
774
'frontend.namespaceCount'.`,
775
)
776
FrontendMaxNamespaceVisibilityRPSPerInstance = NewNamespaceIntSetting(
777
"frontend.namespaceRPS.visibility",
778
10,
779
`FrontendMaxNamespaceVisibilityRPSPerInstance is namespace rate limit per second for visibility APIs.
780
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
781
)
782
FrontendMaxNamespaceNamespaceReplicationInducingAPIsRPSPerInstance = NewNamespaceIntSetting(
783
"frontend.namespaceRPS.namespaceReplicationInducingAPIs",
784
1,
785
`FrontendMaxNamespaceNamespaceReplicationInducingAPIsRPSPerInstance is a per host/per namespace RPS limit for
786
namespace replication inducing APIs (e.g. RegisterNamespace, UpdateNamespace, UpdateWorkerBuildIdCompatibility).
787
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
788
)
789
FrontendMaxNamespaceVisibilityBurstRatioPerInstance = NewNamespaceFloatSetting(
790
"frontend.namespaceBurstRatio.visibility",
791
1,
792
`FrontendMaxNamespaceVisibilityBurstRatioPerInstance is namespace burst limit for visibility APIs as a ratio of
793
namespace visibility RPS. The RPS used here will be the effective RPS from global and per-instance limits. This
794
config is EXPERIMENTAL and may be changed or removed in a later release. The value must be 1 or higher.`,
795
)
796
FrontendMaxNamespaceNamespaceReplicationInducingAPIsBurstRatioPerInstance = NewNamespaceFloatSetting(
797
"frontend.namespaceBurstRatio.namespaceReplicationInducingAPIs",
798
10,
799
`FrontendMaxNamespaceNamespaceReplicationInducingAPIsBurstRatioPerInstance is a per host/per namespace burst limit for
800
namespace replication inducing APIs (e.g. RegisterNamespace, UpdateNamespace, UpdateWorkerBuildIdCompatibility)
801
as a ratio of namespace ReplicationInducingAPIs RPS. The RPS used here will be the effective RPS from global and
802
per-instance limits. This config is EXPERIMENTAL and may be changed or removed in a later release. The value must
803
be 1 or higher.`,
804
)
805
FrontendGlobalNamespaceRPS = NewNamespaceIntSetting(
806
"frontend.globalNamespaceRPS",
807
0,
808
`FrontendGlobalNamespaceRPS is namespace rate limit per second for the whole cluster.
809
The limit is evenly distributed among available frontend service instances.
810
If this is set, it overwrites per instance limit "frontend.namespaceRPS".`,
811
)
812
InternalFrontendGlobalNamespaceRPS = NewNamespaceIntSetting(
813
"internal-frontend.globalNamespaceRPS",
814
0,
815
`InternalFrontendGlobalNamespaceRPS is workflow namespace rate limit per second across
816
all internal-frontends.`,
817
)
818
FrontendGlobalNamespaceVisibilityRPS = NewNamespaceIntSetting(
819
"frontend.globalNamespaceRPS.visibility",
820
0,
821
`FrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second for the whole cluster for visibility API.
822
The limit is evenly distributed among available frontend service instances.
823
If this is set, it overwrites per instance limit "frontend.namespaceRPS.visibility".
824
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
825
)
826
FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS = NewNamespaceIntSetting(
827
"frontend.globalNamespaceRPS.namespaceReplicationInducingAPIs",
828
10,
829
`FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS is a cluster global, per namespace RPS limit for
830
namespace replication inducing APIs (e.g. RegisterNamespace, UpdateNamespace, UpdateWorkerBuildIdCompatibility).
831
The limit is evenly distributed among available frontend service instances.
832
If this is set, it overwrites the per instance limit configured with
833
"frontend.namespaceRPS.namespaceReplicationInducingAPIs".
834
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
835
)
836
InternalFrontendGlobalNamespaceVisibilityRPS = NewNamespaceIntSetting(
837
"internal-frontend.globalNamespaceRPS.visibility",
838
0,
839
`InternalFrontendGlobalNamespaceVisibilityRPS is workflow namespace rate limit per second
840
across all internal-frontends.
841
This config is EXPERIMENTAL and may be changed or removed in a later release.`,
842
)
843
FrontendThrottledLogRPS = NewGlobalIntSetting(
844
"frontend.throttledLogRPS",
845
20,
846
`FrontendThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger`,
847
)
848
FrontendShutdownDrainDuration = NewGlobalDurationSetting(
849
"frontend.shutdownDrainDuration",
850
0*time.Second,
851
`FrontendShutdownDrainDuration is the duration of traffic drain during shutdown`,
852
)
853
FrontendShutdownFailHealthCheckDuration = NewGlobalDurationSetting(
854
"frontend.shutdownFailHealthCheckDuration",
855
0*time.Second,
856
`FrontendShutdownFailHealthCheckDuration is the duration of shutdown failure detection`,
857
)
858
FrontendMaxBadBinaries = NewNamespaceIntSetting(
859
"frontend.maxBadBinaries",
860
10,
861
`FrontendMaxBadBinaries is the max number of bad binaries in namespace config`,
862
)
863
FrontendMaskInternalErrorDetails = NewNamespaceBoolSetting(
864
"frontend.maskInternalErrorDetails",
865
true,
866
`MaskInternalOrUnknownErrors is whether to replace internal/unknown errors with default error`,
867
)
868
FrontendContextMetadataSetTrailer = NewGlobalBoolSetting(
869
"frontend.contextMetadataSetTrailer",
870
false,
871
`FrontendContextMetadataSetTrailer controls whether frontend gRPC handlers emit context metadata in response trailers. This is read when constructing the frontend ContextMetadataInterceptor.`,
872
)
873
HistoryHostErrorPercentage = NewGlobalFloatSetting(
874
"frontend.historyHostErrorPercentage",
875
0.5,
876
`HistoryHostErrorPercentage is the proportion of hosts that are unhealthy through observation external to the host and internal host health checks`,
877
)
878
HistoryHostSelfErrorProportion = NewGlobalFloatSetting(
879
"frontend.historyHostSelfErrorProportion",
880
0.05,
881
`HistoryHostStartingProportion is the proportion of hosts that have marked themselves as not ready -- this could due to waiting to acquire all shards on startup, or an internal health check failure`,
882
)
883
SendRawWorkflowHistory = NewNamespaceBoolSetting(
884
"frontend.sendRawWorkflowHistory",
885
false,
886
`SendRawWorkflowHistory is whether to enable raw history retrieving`,
887
)
888
SearchAttributesNumberOfKeysLimit = NewNamespaceIntSetting(
889
"frontend.searchAttributesNumberOfKeysLimit",
890
100,
891
`SearchAttributesNumberOfKeysLimit is the limit of number of keys`,
892
)
893
SearchAttributesSizeOfValueLimit = NewNamespaceIntSetting(
894
"frontend.searchAttributesSizeOfValueLimit",
895
2*1024,
896
`SearchAttributesSizeOfValueLimit is the size limit of each value`,
897
)
898
SearchAttributesTotalSizeLimit = NewNamespaceIntSetting(
899
"frontend.searchAttributesTotalSizeLimit",
900
40*1024,
901
`SearchAttributesTotalSizeLimit is the size limit of the whole map`,
902
)
903
VisibilityArchivalQueryMaxPageSize = NewGlobalIntSetting(
904
"frontend.visibilityArchivalQueryMaxPageSize",
905
10000,
906
`VisibilityArchivalQueryMaxPageSize is the maximum page size for a visibility archival query`,
907
)
908
EnableServerVersionCheck = NewGlobalBoolSetting(
909
"frontend.enableServerVersionCheck",
910
os.Getenv("TEMPORAL_VERSION_CHECK_DISABLED") == "",
911
`EnableServerVersionCheck is a flag that controls whether or not periodic version checking is enabled`,
912
)
913
EnableTokenNamespaceEnforcement = NewGlobalBoolSetting(
914
"frontend.enableTokenNamespaceEnforcement",
915
true,
916
`EnableTokenNamespaceEnforcement enables enforcement that namespace in completion token matches namespace of the request`,
917
)
918
DisableListVisibilityByFilter = NewNamespaceBoolSetting(
919
"frontend.disableListVisibilityByFilter",
920
false,
921
`DisableListVisibilityByFilter is config to disable list open/close workflow using filter`,
922
)
923
ExposeAuthorizerErrors = NewGlobalBoolSetting(
924
"frontend.exposeAuthorizerErrors",
925
false,
926
`ExposeAuthorizerErrors controls whether the frontend authorization interceptor will pass through errors returned by
927
the Authorizer component. If false, a generic PermissionDenied error without details will be returned. Default false.`,
928
)
929
EnablePrincipalPropagation = NewNamespaceBoolSetting(
930
"frontend.enablePrincipalPropagation",
931
false,
932
`EnablePrincipalPropagation controls whether the authorization interceptor propagates the authenticated
933
principal identity as gRPC headers.`,
934
)
935
KeepAliveMinTime = NewGlobalDurationSetting(
936
"frontend.keepAliveMinTime",
937
10*time.Second,
938
`KeepAliveMinTime is the minimum amount of time a client should wait before sending a keepalive ping.`,
939
)
940
KeepAlivePermitWithoutStream = NewGlobalBoolSetting(
941
"frontend.keepAlivePermitWithoutStream",
942
true,
943
`KeepAlivePermitWithoutStream If true, server allows keepalive pings even when there are no active
944
streams(RPCs). If false, and client sends ping when there are no active
945
streams, server will send GOAWAY and close the connection.`,
946
)
947
KeepAliveMaxConnectionIdle = NewGlobalDurationSetting(
948
"frontend.keepAliveMaxConnectionIdle",
949
2*time.Minute,
950
`KeepAliveMaxConnectionIdle is a duration for the amount of time after which an
951
idle connection would be closed by sending a GoAway. Idleness duration is
952
defined since the most recent time the number of outstanding RPCs became
953
zero or the connection establishment.`,
954
)
955
KeepAliveMaxConnectionAge = NewGlobalDurationSetting(
956
"frontend.keepAliveMaxConnectionAge",
957
5*time.Minute,
958
`KeepAliveMaxConnectionAge is a duration for the maximum amount of time a
959
connection may exist before it will be closed by sending a GoAway. A
960
random jitter of +/-10% will be added to MaxConnectionAge to spread out
961
connection storms.`,
962
)
963
KeepAliveMaxConnectionAgeGrace = NewGlobalDurationSetting(
964
"frontend.keepAliveMaxConnectionAgeGrace",
965
70*time.Second,
966
`KeepAliveMaxConnectionAgeGrace is an additive period after MaxConnectionAge after
967
which the connection will be forcibly closed.`,
968
)
969
KeepAliveTime = NewGlobalDurationSetting(
970
"frontend.keepAliveTime",
971
1*time.Minute,
972
`KeepAliveTime After a duration of this time if the server doesn't see any activity it
973
pings the client to see if the transport is still alive.
974
If set below 1s, a minimum value of 1s will be used instead.`,
975
)
976
KeepAliveTimeout = NewGlobalDurationSetting(
977
"frontend.keepAliveTimeout",
978
10*time.Second,
979
`KeepAliveTimeout After having pinged for keepalive check, the server waits for a duration
980
of Timeout and if no activity is seen even after that the connection is closed.`,
981
)
982
FrontendEnableSchedules = NewNamespaceBoolSetting(
983
"frontend.enableSchedules",
984
true,
985
`FrontendEnableSchedules enables schedule-related RPCs in the frontend`,
986
)
987
// [cleanup-wv-pre-release]
988
EnableDeployments = NewNamespaceBoolSetting(
989
"system.enableDeployments",
990
false,
991
`EnableDeployments enables deployments (deprecated versioning v3 pre-release) in all services,
992
including deployment-related RPCs in the frontend, deployment entity workflows in the worker,
993
and deployment interaction in matching and history.`,
994
)
995
EnableDeploymentVersions = NewNamespaceBoolSetting(
996
"system.enableDeploymentVersions",
997
true,
998
`EnableDeploymentVersions enables deployment versions (versioning v3) in all services,
999
including deployment-related RPCs in the frontend, deployment version entity workflows in the worker,
1000
and deployment interaction in matching and history.`,
1001
)
1002
UseRevisionNumberForWorkerVersioning = NewNamespaceBoolSetting(
1003
"system.useRevisionNumberForWorkerVersioning",
1004
true,
1005
`UseRevisionNumberForWorkerVersioning enables the use of revision number to resolve consistency problems that may arise during task dispatch time.`,
1006
)
1007
EnableSuggestCaNOnNewTargetVersion = NewNamespaceBoolSetting(
1008
"system.enableSuggestCaNOnNewTargetVersion",
1009
false,
1010
`EnableSuggestCaNOnNewTargetVersion lets Pinned workflows receive SuggestContinueAsNew when a new target version is available.`,
1011
)
1012
EnableSendTargetVersionChanged = NewNamespaceBoolSetting(
1013
"system.enableSendTargetVersionChanged",
1014
true,
1015
`EnableSendTargetVersionChanged lets Pinned workflows receive TargetWorkerDeploymentVersionChanged=true when a new target version is available for that workflow.`,
1016
)
1017
AllowDeleteNamespaceIfNexusEndpointTarget = NewGlobalBoolSetting(
1018
"frontend.allowDeleteNamespaceIfNexusEndpointTarget",
1019
false,
1020
`If set to true (default is false), namespaces that are Nexus endpoint targets will be prevented from being deleted.`,
1021
)
1022
1023
RefreshNexusEndpointsLongPollTimeout = NewGlobalDurationSetting(
1024
"system.refreshNexusEndpointsLongPollTimeout",
1025
5*time.Minute,
1026
`RefreshNexusEndpointsLongPollTimeout is the maximum duration of background long poll requests to update Nexus endpoints.`,
1027
)
1028
RefreshNexusEndpointsMinWait = NewGlobalDurationSetting(
1029
"system.refreshNexusEndpointsMinWait",
1030
1*time.Second,
1031
`RefreshNexusEndpointsMinWait is the minimum wait time between background long poll requests to update Nexus endpoints.`,
1032
)
1033
ForceNexusEndpointRefreshOnRead = NewGlobalBoolSetting(
1034
"system.forceNexusEndpointRefreshOnRead",
1035
false,
1036
`ForceNexusEndpointRefreshOnRead forces the Nexus endpoint registry to refresh from matching service on read.
1037
This effectively bypasses the cache so that endpoint writes are visible to readers immediately, instead of after the
1038
next background long-poll refresh. This should not be turned on in production, as it would introduce scalability
1039
and reliability problems.`,
1040
)
1041
NexusReadThroughCacheSize = NewGlobalIntSetting(
1042
"system.nexusReadThroughCacheSize",
1043
100,
1044
`The size of the Nexus endpoint registry's readthrough LRU cache - the cache is a secondary cache and is only
1045
used when the first cache layer has a miss. Requires server restart for change to be applied.`,
1046
)
1047
NexusReadThroughCacheTTL = NewGlobalDurationSetting(
1048
"system.nexusReadThroughCacheTTL",
1049
30*time.Second,
1050
`The TTL of the Nexus endpoint registry's readthrough LRU cache - the cache is a secondary cache and is only
1051
used when the first cache layer has a miss. Requires server restart for change to be applied.`,
1052
)
1053
FrontendNexusRequestHeadersBlacklist = NewGlobalTypedSettingWithConverter(
1054
"frontend.nexusRequestHeadersBlacklist",
1055
ConvertWildcardStringListToRegexp,
1056
// Failure support is an internal implementation detail that shouldn't propagate to the user.
1057
util.MustWildCardStringsToRegexp([]string{
1058
"accept-encoding",
1059
"x-forwarded-for",
1060
"xdc-redirection",
1061
"xdc-redirection-api",
1062
"temporal-nexus-failure-support",
1063
}),
1064
`Nexus request headers to be removed before being sent to a user handler. Wildcards (*) are expanded to
1065
allow any substring. By default headers that are meant for internal use are disallowed. Concrete type should be list of
1066
strings.`,
1067
)
1068
FrontendNexusForwardRequestUseEndpointDispatch = NewGlobalBoolSetting(
1069
"frontend.nexusForwardRequestUseEndpointDispatch",
1070
false,
1071
`!EXPERIMENTAL! NB: This config will be removed in a future release. Controls whether to use Nexus
1072
task dispatch by endpoint URLs for forwarded Nexus requests. If set to true, forwarded requests will use the same
1073
dispatch type (by endpoint or by namespace + task queue) as the original request. If false, dispatch by namespace + task
1074
queue will always be used for forwarded requests. Defaults to false because Nexus endpoints do not support replication,
1075
so forwarding by endpoint ID will not work out of the box.`,
1076
)
1077
FrontendCallbackURLMaxLength = NewNamespaceIntSetting(
1078
"frontend.callbackURLMaxLength",
1079
1000,
1080
`FrontendCallbackURLMaxLength is the maximum length of callback URL`,
1081
)
1082
FrontendCallbackHeaderMaxSize = NewNamespaceIntSetting(
1083
"frontend.callbackHeaderMaxLength",
1084
8*1024,
1085
`FrontendCallbackHeaderMaxSize is the maximum accumulated size of callback header keys and values`,
1086
)
1087
MaxCallbacksPerWorkflow = NewNamespaceIntSetting(
1088
"system.maxCallbacksPerWorkflow",
1089
32,
1090
`MaxCallbacksPerWorkflow is the maximum number of callbacks that can be attached to a workflow.`,
1091
)
1092
MaxCallbacksPerUpdateID = NewNamespaceIntSetting(
1093
"system.maxCallbacksPerUpdateID",
1094
32,
1095
`MaxCallbacksPerUpdateID is the maximum number of callbacks that can be attached to a single update ID.`,
1096
)
1097
FrontendLinkMaxSize = NewNamespaceIntSetting(
1098
"frontend.linkMaxSize",
1099
4000, // Links may include a workflow ID and namespace name, both of which are limited to a length of 1000.
1100
`Maximum size in bytes of temporal.api.common.v1.Link object in an API request.`,
1101
)
1102
FrontendMaxLinksPerRequest = NewNamespaceIntSetting(
1103
"frontend.maxlinksPerRequest",
1104
10,
1105
`Maximum number of links allowed to be attached via a single API request.`,
1106
)
1107
MaxLinksPerComponent = NewNamespaceIntSetting(
1108
"chasm.maxLinksPerComponent",
1109
2000,
1110
`MaxLinksPerComponent is the maximum number of links that can be attached to a single CHASM component (e.g. a standalone activity or standalone Nexus operation) across all start/attach calls.`,
1111
)
1112
FrontendMaxConcurrentBatchOperationPerNamespace = NewNamespaceIntSetting(
1113
"frontend.MaxConcurrentBatchOperationPerNamespace",
1114
1,
1115
`FrontendMaxConcurrentBatchOperationPerNamespace is the max concurrent batch operation job count per namespace`,
1116
)
1117
FrontendMaxExecutionCountBatchOperationPerNamespace = NewNamespaceIntSetting(
1118
"frontend.MaxExecutionCountBatchOperationPerNamespace",
1119
1000,
1120
`FrontendMaxExecutionCountBatchOperationPerNamespace is the max execution count batch operation supports per namespace`,
1121
)
1122
FrontendEnableBatcher = NewNamespaceBoolSetting(
1123
"frontend.enableBatcher",
1124
true,
1125
`FrontendEnableBatcher enables batcher-related RPCs in the frontend`,
1126
)
1127
FrontendMaxConcurrentAdminBatchOperationPerNamespace = NewNamespaceIntSetting(
1128
"frontend.MaxConcurrentAdminBatchOperationPerNamespace",
1129
1,
1130
`FrontendMaxConcurrentAdminBatchOperationPerNamespace is the max concurrent admin batch operation job count per namespace`,
1131
)
1132
FrontendEnableBatchOperationsForStandaloneActivities = NewNamespaceBoolSetting(
1133
"frontend.enableBatchOperationsForStandaloneActivities",
1134
false,
1135
`FrontendEnableBatchOperationsForStandaloneActivities controls whether the frontend accepts the batch cancel, terminate, and delete standalone activity operation fields`,
1136
)
1137
1138
FrontendEnableUpdateWorkflowExecution = NewNamespaceBoolSetting(
1139
"frontend.enableUpdateWorkflowExecution",
1140
true,
1141
`FrontendEnableUpdateWorkflowExecution enables UpdateWorkflowExecution API in the frontend.`,
1142
)
1143
1144
FrontendEnableUpdateWorkflowExecutionAsyncAccepted = NewNamespaceBoolSetting(
1145
"frontend.enableUpdateWorkflowExecutionAsyncAccepted",
1146
true,
1147
`FrontendEnableUpdateWorkflowExecutionAsyncAccepted enables the UpdateWorkflowExecution API
1148
to allow waiting on the "Accepted" lifecycle stage.`,
1149
)
1150
1151
FrontendEnableWorkerVersioningDataAPIs = NewNamespaceBoolSetting(
1152
"frontend.workerVersioningDataAPIs",
1153
false,
1154
`FrontendEnableWorkerVersioningDataAPIs enables worker versioning data read / write APIs.`,
1155
)
1156
FrontendEnableWorkerVersioningWorkflowAPIs = NewNamespaceBoolSetting(
1157
"frontend.workerVersioningWorkflowAPIs",
1158
true,
1159
`FrontendEnableWorkerVersioningWorkflowAPIs enables worker versioning in workflow progress APIs.`,
1160
)
1161
FrontendEnableWorkerVersioningRuleAPIs = NewNamespaceBoolSetting(
1162
"frontend.workerVersioningRuleAPIs",
1163
false,
1164
`FrontendEnableWorkerVersioningRuleAPIs enables worker versioning in workflow progress APIs.`,
1165
)
1166
1167
DeleteNamespaceUseChasmDeleteExecution = NewGlobalBoolSetting(
1168
"frontend.deleteNamespaceUseChasmDeleteExecution",
1169
false,
1170
`DeleteNamespaceUseChasmDeleteExecution controls whether the delete namespace workflow uses the
1171
DeleteExecution history service API (CHASM engine path) for non-workflow CHASM executions, instead
1172
of ForceDeleteWorkflowExecution. Only enable after all history and worker services have been upgraded
1173
to a version that supports the DeleteExecution API.`,
1174
)
1175
1176
DeleteNamespaceDeleteActivityRPS = NewGlobalIntSetting(
1177
"frontend.deleteNamespaceDeleteActivityRPS",
1178
100,
1179
`DeleteNamespaceDeleteActivityRPS is an RPS per every parallel delete executions activity.
1180
Total RPS is equal to DeleteNamespaceDeleteActivityRPS * DeleteNamespaceConcurrentDeleteExecutionsActivities.
1181
Default value is 100. Despite starting with 'frontend.' this setting is used by a worker and can be changed while namespace is deleted.`,
1182
)
1183
DeleteNamespacePageSize = NewGlobalIntSetting(
1184
"frontend.deleteNamespaceDeletePageSize",
1185
1000,
1186
`DeleteNamespacePageSize is a page size to read executions from visibility for delete executions activity.
1187
Default value is 1000. Read once before delete of specified namespace is started.`,
1188
)
1189
DeleteNamespacePagesPerExecution = NewGlobalIntSetting(
1190
"frontend.deleteNamespacePagesPerExecution",
1191
256,
1192
`DeleteNamespacePagesPerExecution is a number of pages before returning ContinueAsNew from delete executions activity.
1193
Default value is 256. Read once before delete of specified namespace is started.`,
1194
)
1195
DeleteNamespaceConcurrentDeleteExecutionsActivities = NewGlobalIntSetting(
1196
"frontend.deleteNamespaceConcurrentDeleteExecutionsActivities",
1197
4,
1198
`DeleteNamespaceConcurrentDeleteExecutionsActivities is a number of concurrent delete executions activities.
1199
Must be not greater than 256 and number of worker cores in the cluster.
1200
Default is 4. Read once before delete of specified namespace is started.`,
1201
)
1202
DeleteNamespaceNamespaceDeleteDelay = NewGlobalDurationSetting(
1203
"frontend.deleteNamespaceNamespaceDeleteDelay",
1204
0*time.Hour,
1205
`DeleteNamespaceNamespaceDeleteDelay is a duration for how long namespace stays in database
1206
after all namespace resources (i.e. workflow executions) are deleted.
1207
Default is 0, means, namespace will be deleted immediately.`,
1208
)
1209
ProtectedNamespaces = NewGlobalTypedSetting(
1210
"worker.protectedNamespaces",
1211
([]string)(nil),
1212
`List of namespace names that can't be deleted.`,
1213
)
1214
1215
// keys for matching
1216
1217
MatchingRPS = NewGlobalIntSetting(
1218
"matching.rps",
1219
1200,
1220
`MatchingRPS is request rate per second for each matching host`,
1221
)
1222
MatchingNamespaceRPS = NewNamespaceIntSetting(
1223
"matching.namespaceRPS",
1224
0,
1225
`MatchingNamespaceRPS is namespace rate limit per second for each matching host.
1226
If value less or equal to 0, will fall back to MatchingRPS`,
1227
)
1228
MatchingPersistenceMaxQPS = NewGlobalIntSetting(
1229
"matching.persistenceMaxQPS",
1230
3000,
1231
`MatchingPersistenceMaxQPS is the max qps matching host can query DB`,
1232
)
1233
MatchingPersistenceGlobalMaxQPS = NewGlobalIntSetting(
1234
"matching.persistenceGlobalMaxQPS",
1235
0,
1236
`MatchingPersistenceGlobalMaxQPS is the max qps matching cluster can query DB`,
1237
)
1238
MatchingPersistenceNamespaceMaxQPS = NewNamespaceIntSetting(
1239
"matching.persistenceNamespaceMaxQPS",
1240
0,
1241
`MatchingPersistenceNamespaceMaxQPS is the max qps each namespace on matching host can query DB`,
1242
)
1243
MatchingPersistenceGlobalNamespaceMaxQPS = NewNamespaceIntSetting(
1244
"matching.persistenceGlobalNamespaceMaxQPS",
1245
0,
1246
`MatchingPersistenceNamespaceMaxQPS is the max qps each namespace in matching cluster can query DB`,
1247
)
1248
MatchingPersistenceDynamicRateLimitingParams = NewGlobalTypedSetting(
1249
"matching.persistenceDynamicRateLimitingParams",
1250
DefaultDynamicRateLimitingParams,
1251
`MatchingPersistenceDynamicRateLimitingParams is a struct that contains all adjustable dynamic rate limiting params.
1252
Fields: Enabled, RefreshInterval, LatencyThreshold, ErrorThreshold, RateBackoffStepSize, RateIncreaseStepSize, RateMultiMin, RateMultiMax.
1253
See DynamicRateLimitingParams comments for more details.`,
1254
)
1255
MatchingMinTaskThrottlingBurstSize = NewTaskQueueIntSetting(
1256
"matching.minTaskThrottlingBurstSize",
1257
1,
1258
`MatchingMinTaskThrottlingBurstSize is the minimum burst size for task queue throttling`,
1259
)
1260
MatchingGetTasksBatchSize = NewTaskQueueIntSetting(
1261
"matching.getTasksBatchSize",
1262
1000,
1263
`How many backlog tasks to read from persistence at once`,
1264
)
1265
MatchingGetTasksReloadAt = NewTaskQueueIntSetting(
1266
"matching.getTasksReloadAt",
1267
100,
1268
`Reload a batch of tasks when there are this many remaining. Must be less than MatchingGetTasksBatchSize. (Requires new matcher.)`,
1269
)
1270
MatchingLongPollExpirationInterval = NewTaskQueueDurationSetting(
1271
"matching.longPollExpirationInterval",
1272
time.Minute,
1273
`MatchingLongPollExpirationInterval is the long poll expiration interval in the matching service`,
1274
)
1275
// TODO(pri): old matcher cleanup
1276
MatchingSyncMatchWaitDuration = NewTaskQueueDurationSetting(
1277
"matching.syncMatchWaitDuration",
1278
200*time.Millisecond,
1279
`MatchingSyncMatchWaitDuration is to wait time for sync match`,
1280
)
1281
MatchingHistoryMaxPageSize = NewNamespaceIntSetting(
1282
"matching.historyMaxPageSize",
1283
primitives.GetHistoryMaxPageSize,
1284
`MatchingHistoryMaxPageSize is the maximum page size of history events returned on PollWorkflowTaskQueue requests`,
1285
)
1286
MatchingUpdateAckInterval = NewTaskQueueDurationSettingWithConstrainedDefault(
1287
"matching.updateAckInterval",
1288
[]TypedConstrainedValue[time.Duration]{
1289
// Use a longer default interval for the per-namespace internal worker queues.
1290
{
1291
Constraints: Constraints{
1292
TaskQueueName: primitives.PerNSWorkerTaskQueue,
1293
},
1294
Value: 5 * time.Minute,
1295
},
1296
// Default for everything else.
1297
{
1298
Value: 1 * time.Minute,
1299
},
1300
},
1301
`MatchingUpdateAckInterval is the interval for update ack`,
1302
)
1303
MatchingMetadataUpdateOnAppendInterval = NewTaskQueueDurationSetting(
1304
"matching.metadataUpdateOnAppendInterval",
1305
5*time.Second,
1306
`MatchingMetadataUpdateOnAppendInterval controls how often task queue metadata (e.g.
1307
approximate backlog count) is written along with task appends. When using Cassandra, task appends
1308
always require an LWT for the range ID check, but updating the full metadata on every append adds
1309
extra write cost. This setting limits metadata updates to at most once per interval, piggybacking
1310
on the append LWT. A value of 0 means always update metadata on every append (previous behavior).`,
1311
)
1312
MatchingMaxTaskQueueIdleTime = NewTaskQueueDurationSetting(
1313
"matching.maxTaskQueueIdleTime",
1314
5*time.Minute,
1315
`MatchingMaxTaskQueueIdleTime is the time after which an idle task queue will be unloaded.
1316
Note: this should be greater than matching.longPollExpirationInterval and matching.getUserDataLongPollTimeout.`,
1317
)
1318
MatchingOutstandingTaskAppendsThreshold = NewTaskQueueIntSetting(
1319
"matching.outstandingTaskAppendsThreshold",
1320
250,
1321
`MatchingOutstandingTaskAppendsThreshold is the threshold for outstanding task appends`,
1322
)
1323
MatchingMaxTaskBatchSize = NewTaskQueueIntSetting(
1324
"matching.maxTaskBatchSize",
1325
100,
1326
`MatchingMaxTaskBatchSize is max batch size for task writer`,
1327
)
1328
MatchingMaxTaskDeleteBatchSize = NewTaskQueueIntSetting(
1329
"matching.maxTaskDeleteBatchSize",
1330
100,
1331
`MatchingMaxTaskDeleteBatchSize is the max batch size for range deletion of tasks`,
1332
)
1333
MatchingTaskDeleteInterval = NewTaskQueueDurationSetting(
1334
"matching.taskDeleteInterval",
1335
15*time.Second,
1336
`MatchingTaskDeleteInterval is the minimum interval between task range deletions`,
1337
)
1338
MatchingThrottledLogRPS = NewGlobalIntSetting(
1339
"matching.throttledLogRPS",
1340
20,
1341
`MatchingThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger`,
1342
)
1343
MatchingNumTaskqueueWritePartitions = NewTaskQueueIntSettingWithConstrainedDefault(
1344
"matching.numTaskqueueWritePartitions",
1345
defaultNumTaskQueuePartitions,
1346
`MatchingNumTaskqueueWritePartitions is the number of write partitions for a task queue`,
1347
)
1348
MatchingNumTaskqueueReadPartitions = NewTaskQueueIntSettingWithConstrainedDefault(
1349
"matching.numTaskqueueReadPartitions",
1350
defaultNumTaskQueuePartitions,
1351
`MatchingNumTaskqueueReadPartitions is the number of read partitions for a task queue`,
1352
)
1353
MetricsBreakdownByTaskQueue = NewTaskQueueBoolSetting(
1354
"metrics.breakdownByTaskQueue",
1355
true,
1356
`MetricsBreakdownByTaskQueue determines if the 'taskqueue' tag in Matching and History metrics should
1357
contain the actual TQ name or a generic __omitted__ value. Disable this option if the cardinality is too high for your
1358
observability stack. Disabling this option will disable all the per-Task Queue gauges such as backlog lag, count, and age.`,
1359
)
1360
MetricsBreakdownByPartition = NewTaskQueueBoolSetting(
1361
"metrics.breakdownByPartition",
1362
true,
1363
`MetricsBreakdownByPartition determines if the 'partition' tag in Matching metrics should
1364
contain the actual normal partition ID or a generic __normal__ value. Regardless of this config, the tag value for sticky
1365
queues will be "__sticky__". Disable this option if the partition cardinality is too high for your
1366
observability stack. Disabling this option will disable all the per-Task Queue gauges such as backlog lag, count, and age.`,
1367
)
1368
MetricsBreakdownByBuildID = NewTaskQueueBoolSetting(
1369
"metrics.breakdownByBuildID",
1370
true,
1371
`MetricsBreakdownByBuildID determines if the 'worker_version' tag in Matching metrics should
1372
contain the actual Worker Deployment Version or a generic "__versioned__" value. Regardless of this config, the tag value for unversioned
1373
queues will be "__unversioned__". Disable this option if the version cardinality is too high for your
1374
observability stack. Disabling this option will disable all the per-Task Queue gauges such as backlog lag, count, and age
1375
for VERSIONED queues.`,
1376
)
1377
MatchingForwarderMaxOutstandingPolls = NewTaskQueueIntSetting(
1378
"matching.forwarderMaxOutstandingPolls",
1379
1,
1380
`MatchingForwarderMaxOutstandingPolls is the max number of inflight polls from the forwarder`,
1381
)
1382
MatchingForwarderMaxOutstandingTasks = NewTaskQueueIntSetting(
1383
"matching.forwarderMaxOutstandingTasks",
1384
1,
1385
`MatchingForwarderMaxOutstandingTasks is the max number of inflight addTask/queryTask from the forwarder`,
1386
)
1387
MatchingForwarderMaxRatePerSecond = NewTaskQueueFloatSetting(
1388
"matching.forwarderMaxRatePerSecond",
1389
10,
1390
`MatchingForwarderMaxRatePerSecond is the max rate at which add/query can be forwarded`,
1391
)
1392
MatchingForwarderMaxChildrenPerNode = NewTaskQueueIntSetting(
1393
"matching.forwarderMaxChildrenPerNode",
1394
20,
1395
`MatchingForwarderMaxChildrenPerNode is the max number of children per node in the task queue partition tree`,
1396
)
1397
MatchingAlignMembershipChange = NewGlobalDurationSetting(
1398
"matching.alignMembershipChange",
1399
0*time.Second,
1400
`MatchingAlignMembershipChange is a duration to align matching's membership changes to.
1401
This can help reduce effects of task queue movement.`,
1402
)
1403
MatchingShutdownDrainDuration = NewGlobalDurationSetting(
1404
"matching.shutdownDrainDuration",
1405
0*time.Second,
1406
`MatchingShutdownDrainDuration is the duration of traffic drain during shutdown`,
1407
)
1408
MatchingGetUserDataLongPollTimeout = NewGlobalDurationSetting(
1409
"matching.getUserDataLongPollTimeout",
1410
5*time.Minute-10*time.Second,
1411
`MatchingGetUserDataLongPollTimeout is the max length of long polls for GetUserData calls between partitions.`,
1412
)
1413
MatchingGetUserDataRefresh = NewGlobalDurationSetting(
1414
"matching.getUserDataRefresh",
1415
5*time.Minute,
1416
`MatchingGetUserDataRefresh is how often the user data owner refreshes data from persistence.`,
1417
)
1418
MatchingEphemeralDataUpdateInterval = NewTaskQueueDurationSetting(
1419
"matching.ephemeralDataUpdateInterval",
1420
5*time.Second,
1421
`How often to update ephemeral data (e.g. backlog size for forwarding sticky polls).
1422
Set to zero to disable ephemeral data updates.`,
1423
)
1424
MatchingBacklogMetricsEmitInterval = NewTaskQueueDurationSetting(
1425
"matching.backlogMetricsEmitInterval",
1426
time.Minute,
1427
`How often to emit version-attributed backlog metrics. Done on an interval because accurate attribution requires checking the routing config of a task queue to correctly attribute the default queue's tasks to the appropriate current or ramping versions. Set to zero to disable version-attributed backlog metrics.`,
1428
)
1429
MatchingPriorityBacklogForwarding = NewTaskQueueBoolSetting(
1430
"matching.priorityBacklogForwarding",
1431
true,
1432
`Whether to forward polls to partitions with higher-priority backlog.`,
1433
)
1434
MatchingBacklogNegligibleAge = NewTaskQueueDurationSetting(
1435
"matching.backlogNegligibleAge",
1436
5*time.Second,
1437
`MatchingBacklogNegligibleAge is a threshold for negligible vs significant backlogs:
1438
If the head of the backlog is older than this, then we stop sync match and forwarding to ensure
1439
more equal dispatch order among partitions. We also forward sticky polls to partitions with
1440
higher-priority backlog.`,
1441
)
1442
MatchingMaxWaitForPollerBeforeFwd = NewTaskQueueDurationSetting(
1443
"matching.maxWaitForPollerBeforeFwd",
1444
200*time.Millisecond,
1445
`MatchingMaxWaitForPollerBeforeFwd in presence of a non-negligible backlog, we resume forwarding tasks if the
1446
duration since last poll exceeds this threshold.`,
1447
)
1448
QueryPollerUnavailableWindow = NewGlobalDurationSetting(
1449
"matching.queryPollerUnavailableWindow",
1450
20*time.Second,
1451
`QueryPollerUnavailableWindow WF Queries are rejected after a while if no poller has been seen within the window`,
1452
)
1453
WorkerControllerNoPollerHookWindow = NewGlobalDurationSetting(
1454
"matching.workerControllerNoPollerHookWindow",
1455
5*time.Second,
1456
`WorkerControllerNoPollerHookWindow controls how recently a worker must have polled before skipping the WCI scale-up signal on an incoming query or Nexus task dispatch`,
1457
)
1458
MatchingEmitTaskDispatchLatencyAtPoll = NewTaskQueueBoolSetting(
1459
"matching.emitTaskDispatchLatencyAtPoll",
1460
true,
1461
`When enabled, TaskDispatchLatencyPerTaskQueue is emitted when responding to poll requests (with extra tags
1462
like partition and worker-version) instead of being emitted at the matcher level.`,
1463
)
1464
MatchingListNexusEndpointsLongPollTimeout = NewGlobalDurationSetting(
1465
"matching.listNexusEndpointsLongPollTimeout",
1466
5*time.Minute-10*time.Second,
1467
`MatchingListNexusEndpointsLongPollTimeout is the max length of long polls for ListNexusEndpoints calls.`,
1468
)
1469
MatchingNexusEndpointsRefreshInterval = NewGlobalDurationSetting(
1470
"matching.nexusEndpointsRefreshInterval",
1471
10*time.Second,
1472
`Time to wait between calls to check that the in-memory view of Nexus endpoints matches the persisted state.`,
1473
)
1474
MatchingMembershipUnloadDelay = NewGlobalDurationSetting(
1475
"matching.membershipUnloadDelay",
1476
500*time.Millisecond,
1477
`MatchingMembershipUnloadDelay is how long to wait to re-confirm loss of ownership before unloading a task queue.
1478
Set to zero to disable proactive unload.`,
1479
)
1480
MatchingQueryWorkflowTaskTimeoutLogRate = NewTaskQueueFloatSetting(
1481
"matching.queryWorkflowTaskTimeoutLogRate",
1482
0.0,
1483
`MatchingQueryWorkflowTaskTimeoutLogRate defines the sampling rate for logs when a query workflow task times out. Since
1484
these log lines can be noisy, we want to be able to turn on and sample selectively for each affected namespace.`,
1485
)
1486
TaskQueueInfoByBuildIdTTL = NewTaskQueueDurationSetting(
1487
"matching.TaskQueueInfoByBuildIdTTL",
1488
5*time.Second,
1489
`TaskQueueInfoByBuildIdTTL serves as a TTL for the cache holding DescribeTaskQueue partition results`,
1490
)
1491
MatchingDeploymentWorkflowVersion = NewNamespaceIntSetting(
1492
"matching.deploymentWorkflowVersion",
1493
2,
1494
`MatchingDeploymentWorkflowVersion controls what version of the logic should the manager workflows use.`,
1495
)
1496
MatchingMaxTaskQueuesInDeployment = NewNamespaceIntSetting(
1497
"matching.maxTaskQueuesInDeployment",
1498
1000,
1499
`MatchingMaxTaskQueuesInDeployment represents the maximum number of task-queues that can be registed in a single deployment`,
1500
)
1501
MatchingMaxDeployments = NewNamespaceIntSetting(
1502
"matching.maxDeployments",
1503
100,
1504
`MatchingMaxDeployments represents the maximum number of worker deployments that can be registered in a single namespace`,
1505
)
1506
MatchingMaxVersionsInDeployment = NewNamespaceIntSetting(
1507
"matching.maxVersionsInDeployment",
1508
100,
1509
`MatchingMaxVersionsInDeployment represents the maximum number of versions that can be registered in a single worker deployment`,
1510
)
1511
MatchingMaxVersionsInTaskQueue = NewNamespaceIntSetting(
1512
"matching.maxVersionsInTaskQueue",
1513
200,
1514
`MatchingMaxVersionsInTaskQueue represents the maximum number of versions that can be registered in a single task queue.
1515
Should be larger than MatchingMaxVersionsInDeployment because a task queue can be in versions spanning across more than one deployments.`,
1516
)
1517
MatchingMaxTaskQueuesInDeploymentVersion = NewNamespaceIntSetting(
1518
"matching.maxTaskQueuesInDeploymentVersion",
1519
100,
1520
`MatchingMaxTaskQueuesInDeployment represents the maximum number of task-queues that can be registered in a single worker deployment version`,
1521
)
1522
MatchingPollerScalingBacklogAgeScaleUp = NewTaskQueueDurationSetting(
1523
"matching.pollerScalingMinimumBacklog",
1524
200*time.Millisecond,
1525
`MatchingPollerScalingBacklogAgeScaleUp is the minimum backlog age that must be accumulated before
1526
a decision to scale up the number of pollers will be issued`,
1527
)
1528
MatchingPollerScalingWaitTime = NewTaskQueueDurationSetting(
1529
"matching.pollerScalingWaitTime",
1530
1*time.Second,
1531
`MatchingPollerScalingWaitTime is the duration a sync-matched poller must exceed before
1532
a decision to scale down the number of pollers will be issued`,
1533
)
1534
MatchingPollerScalingDecisionsPerSecond = NewTaskQueueFloatSetting(
1535
"matching.pollerScalingDecisionsPerSecond",
1536
10,
1537
`MatchingPollerScalingDecisionsPerSecond is the maximum number of scaling decisions that will be issued per
1538
second per poller by one physical queue manager`,
1539
)
1540
MatchingPollerScalingTaskAddToDispatchRatio = NewTaskQueueFloatSetting(
1541
"matching.pollerScalingTaskAddToDispatchRatio",
1542
1.2,
1543
`MatchingPollerScalingTaskAddToDispatchRatio is the ratio of task add rate to task
1544
dispatch rate above which a decision to scale up the number of pollers will be issued`,
1545
)
1546
MatchingEnablePollerScalingDecisionMetrics = NewTaskQueueBoolSetting(
1547
"matching.enablePollerScalingDecisionMetrics",
1548
false,
1549
`MatchingEnablePollerScalingDecisionMetrics, when enabled, causes matching to emit the poller_scale_decision
1550
metric describing why pollers are scaled up, down, or held for a physical task queue. This is opt-in and can be
1551
scoped by namespace and/or task queue.`,
1552
)
1553
MatchingUseNewMatcher = NewTaskQueueTypedSettingWithConverter(
1554
"matching.useNewMatcher",
1555
ConvertGradualChange(true),
1556
StaticGradualChange(true),
1557
`Use priority-enabled TaskMatcher`,
1558
)
1559
MatchingEnableFairness = NewTaskQueueTypedSettingWithConverter(
1560
"matching.enableFairness",
1561
ConvertGradualChange(false),
1562
StaticGradualChange(false),
1563
`Enable fairness for task dispatching. Implies matching.useNewMatcher.`,
1564
)
1565
MatchingEnableMigration = NewTaskQueueBoolSetting(
1566
"matching.enableMigration",
1567
true,
1568
`Allows migration between v1 and v2 (fairness) task backlogs.`,
1569
)
1570
MatchingPriorityLevels = NewTaskQueueIntSetting(
1571
"matching.priorityLevels",
1572
5,
1573
`Number of simple priority levels (requires new matcher)`,
1574
)
1575
MatchingBacklogTaskForwardTimeout = NewTaskQueueDurationSetting(
1576
"matching.backlogTaskForwardTimeout",
1577
60*time.Second,
1578
`Timeout for forwarded backlog task (requires new matcher)`,
1579
)
1580
MatchingForwardPollRetryMaxInterval = NewTaskQueueDurationSetting(
1581
"matching.forwardPollRetryMaxInterval",
1582
10*time.Second,
1583
`Max backoff interval when retrying a rate-limited ForwardPoll from a child partition`,
1584
)
1585
MatchingFairnessCounter = NewTaskQueueTypedSetting(
1586
"matching.fairnessCounter",
1587
counter.DefaultCounterParams,
1588
`Configuration for counter used in matching fairness.`,
1589
)
1590
MatchingFairnessPassDither = NewTaskQueueBoolSetting(
1591
"matching.fairnessPassDither",
1592
false,
1593
`When true, dither the starting pass of new/reset fairness keys over their initial
1594
stride instead of starting them all at the ack level. This spreads low-weight keys ahead
1595
in pass-space so they don't clump at the front after a counter reset (e.g. partition
1596
movement), at the cost of cross-key FIFO ordering for bursts of equal-weight new keys.`,
1597
)
1598
MatchingFairnessKeyRateLimitCacheSize = NewTaskQueueIntSetting(
1599
"matching.fairnessKeyRateLimitCacheSize",
1600
2000,
1601
"Cache size for fairness key rate limits.",
1602
)
1603
MatchingMaxFairnessKeyWeightOverrides = NewTaskQueueIntSetting(
1604
"matching.maxFairnessKeyWeightOverrides",
1605
1000,
1606
"Maximum number of fairness key weight overrides that can be configured for a task queue at a time.",
1607
)
1608
MatchingEnableWorkerPluginMetrics = NewGlobalBoolSetting(
1609
"matching.enableWorkerPluginMetrics",
1610
false,
1611
`MatchingEnableWorkerPluginMetrics controls whether to export worker plugin metrics.
1612
The metric has 2 dimensions: namespace_id and plugin_name. Disabled by default as this is
1613
an optional feature and also requires a metrics collection system that can handle higher cardinalities.`,
1614
)
1615
MatchingEnablePollerAutoscalingMetrics = NewGlobalBoolSetting(
1616
"matching.enablePollerAutoscalingMetrics",
1617
false,
1618
`MatchingEnablePollerAutoscalingMetrics controls whether to export poller autoscaling metrics.
1619
The metric has dimensions: namespace, taskqueue, and task_type (Workflow, Activity, Nexus). Disabled by
1620
default as namespace cardinality can be high and this requires a metrics collection system that can handle it.`,
1621
)
1622
MatchingAutoEnableV2 = NewTaskQueueBoolSetting(
1623
"matching.autoEnableV2",
1624
false,
1625
`MatchingAutoEnableV2 automatically enables fairness when a fairness or priority key is seen`,
1626
)
1627
MatchingPartitionScaleAllowedDrift = NewTaskQueueTypedSetting(
1628
"matching.partitionScaleAllowedDrift",
1629
PartitionScaleAllowedDrift{
1630
Delta: 1,
1631
Ratio: 1.5,
1632
},
1633
`How far off client partition scale values have to be to reject RPCs.`,
1634
)
1635
MatchingPartitionScaleManager = NewTaskQueueTypedSetting(
1636
"matching.partitionScaleManager",
1637
PartitionScaleManagerSettings{
1638
MaxRate: 0.33,
1639
ShrinkRatio: 0.1,
1640
ShrinkDelta: 8,
1641
BatchSize: 100,
1642
BackgroundInterval: 23 * time.Second,
1643
DrainBufferTime: 15 * time.Second,
1644
ShadowModeLogInterval: 0,
1645
},
1646
`Settings for partition scale manager.`,
1647
)
1648
MatchingPartitionScaler = NewTaskQueueTypedSettingWithConverter(
1649
"matching.partitionScaler",
1650
ConvertSimplePartitionScalerSettings,
1651
SimplePartitionScalerSettings{},
1652
`Settings for simple partition scaler.`,
1653
)
1654
1655
MatchingForceReadTasksOnWrite = NewTaskQueueBoolSetting(
1656
"matching.forceReadTasksOnWrite",
1657
false,
1658
`When true and the fair task reader detects a stuck state (atEnd=false, loadedTasks=0, no
1659
read goroutine running), the write path calls maybeReadTasksLocked to attempt to unblock it.
1660
This is a diagnostic flag — the root cause of the stuck state is still under investigation.`,
1661
)
1662
1663
// Worker registry settings
1664
MatchingWorkerRegistryNumBuckets = NewGlobalIntSetting(
1665
"matching.workerRegistryNumBuckets",
1666
10,
1667
`MatchingWorkerRegistryNumBuckets is the number of buckets used to partition the worker registry
1668
keyspace for reduced lock contention. Changes require a restart to take effect.`,
1669
)
1670
MatchingWorkerRegistryEntryTTL = NewGlobalDurationSetting(
1671
"matching.workerRegistryEntryTTL",
1672
5*time.Minute,
1673
`MatchingWorkerRegistryEntryTTL is the time after which worker heartbeat entries are considered expired
1674
and eligible for eviction. Workers typically heartbeat every 30-60 seconds, so 5 minutes without a
1675
heartbeat indicates the worker is likely dead.`,
1676
)
1677
MatchingWorkerRegistryMinEvictAge = NewGlobalDurationSetting(
1678
"matching.workerRegistryMinEvictAge",
1679
1*time.Minute,
1680
`MatchingWorkerRegistryMinEvictAge is the minimum age of worker heartbeat entries before they can be
1681
evicted due to capacity pressure. This prevents evicting recently-heartbeated workers even when
1682
the registry is at capacity. Lower values help handle crash-looping workers more aggressively.`,
1683
)
1684
MatchingWorkerRegistryMaxEntries = NewGlobalIntSetting(
1685
"matching.workerRegistryMaxEntries",
1686
1_000_000,
1687
`MatchingWorkerRegistryMaxEntries is the maximum number of worker heartbeat entries allowed across
1688
all namespaces. When exceeded, the oldest entries (older than MinEvictAge) are evicted.`,
1689
)
1690
MatchingWorkerRegistryEvictionInterval = NewGlobalDurationSetting(
1691
"matching.workerRegistryEvictionInterval",
1692
1*time.Minute,
1693
`MatchingWorkerRegistryEvictionInterval is how often the worker registry runs background eviction
1694
to remove expired entries. Should be shorter than EntryTTL for timely cleanup. Lower values mean faster cleanup but more CPU overhead.`,
1695
)
1696
MatchingSpreadRoutingBatchSize = NewGlobalTypedSettingWithConverter(
1697
"matching.spreadRoutingBatchSize",
1698
ConvertGradualChange[int](0),
1699
StaticGradualChange[int](0),
1700
`If non-zero, try to spread task queue partitions across matching nodes better, using the given batch size.
1701
Don't change this on a live cluster without using the gradual change mechanism.
1702
`,
1703
)
1704
MatchingConnectionCloseDelay = NewGlobalDurationSetting(
1705
"matching.connectionCloseDelay",
1706
30*time.Second,
1707
`MatchingConnectionCloseDelay delays closing a cached matching client connection after its host
1708
leaves the membership ring, giving in-flight long-polls time to drain before the connection is closed.`,
1709
)
1710
1711
// keys for history
1712
1713
EnableReplicationStream = NewGlobalBoolSetting(
1714
"history.enableReplicationStream",
1715
true,
1716
`EnableReplicationStream turn on replication stream`,
1717
)
1718
EmitReplicationLifecycleEvents = NewGlobalBoolSetting(
1719
"history.emitReplicationLifecycleEvents",
1720
false,
1721
`EmitReplicationLifecycleEvents controls whether the history service emits ReplicationLifecycle wide events (sent/executing/applied phases). Cluster-level; default off.`,
1722
)
1723
EnableCloseInboundReplicationStreamOnShutdown = NewGlobalBoolSetting(
1724
"history.enableCloseInboundReplicationStreamOnShutdown",
1725
true,
1726
`EnableCloseInboundReplicationStreamOnShutdown closes inbound replication streams on shutdown, signaling the remote sender to stop. Disable if this causes unexpected issues during rolling restarts.`,
1727
)
1728
EnableSeparateReplicationEnableFlag = NewGlobalBoolSetting(
1729
"history.enableSeparateReplicationEnableFlag",
1730
false,
1731
`EnableSeparateReplicationEnableFlag controls whether to use the new ReplicationEnabled flag to control replication streams separately from cluster connectivity. When false, falls back to using only the Enabled flag for both connectivity and replication.`,
1732
)
1733
EnableHistoryReplicationDLQV2 = NewGlobalBoolSetting(
1734
"history.enableHistoryReplicationDLQV2",
1735
true,
1736
`EnableHistoryReplicationDLQV2 switches to the DLQ v2 implementation for history replication. See details in
1737
[go.temporal.io/server/common/persistence.QueueV2]`,
1738
)
1739
1740
HistoryRPS = NewGlobalIntSetting(
1741
"history.rps",
1742
3000,
1743
`HistoryRPS is request rate per second for each history host`,
1744
)
1745
HistoryNamespaceRPS = NewNamespaceIntSetting(
1746
"history.namespaceRPS",
1747
0,
1748
`HistoryNamespaceRPS is namespace rate limit per second for each history host.
1749
If value less or equal to 0, will fall back to HistoryRPS`,
1750
)
1751
HistoryPersistenceMaxQPS = NewGlobalIntSetting(
1752
"history.persistenceMaxQPS",
1753
9000,
1754
`HistoryPersistenceMaxQPS is the max qps history host can query DB`,
1755
)
1756
HistoryPersistenceGlobalMaxQPS = NewGlobalIntSetting(
1757
"history.persistenceGlobalMaxQPS",
1758
0,
1759
`HistoryPersistenceGlobalMaxQPS is the max qps history cluster can query DB`,
1760
)
1761
HistoryPersistenceNamespaceMaxQPS = NewNamespaceIntSetting(
1762
"history.persistenceNamespaceMaxQPS",
1763
0,
1764
`HistoryPersistenceNamespaceMaxQPS is the max qps each namespace on history host can query DB
1765
If value less or equal to 0, will fall back to HistoryPersistenceMaxQPS`,
1766
)
1767
HistoryPersistenceGlobalNamespaceMaxQPS = NewNamespaceIntSetting(
1768
"history.persistenceGlobalNamespaceMaxQPS",
1769
0,
1770
`HistoryPersistenceNamespaceMaxQPS is the max qps each namespace in history cluster can query DB`,
1771
)
1772
HistoryPersistencePerShardNamespaceMaxQPS = NewNamespaceIntSetting(
1773
"history.persistencePerShardNamespaceMaxQPS",
1774
0,
1775
`HistoryPersistencePerShardNamespaceMaxQPS is the max qps each namespace on a shard can query DB`,
1776
)
1777
HistoryPersistenceDynamicRateLimitingParams = NewGlobalTypedSetting(
1778
"history.persistenceDynamicRateLimitingParams",
1779
DefaultDynamicRateLimitingParams,
1780
`HistoryPersistenceDynamicRateLimitingParams is a struct that contains all adjustable dynamic rate limiting params.
1781
Fields: Enabled, RefreshInterval, LatencyThreshold, ErrorThreshold, RateBackoffStepSize, RateIncreaseStepSize, RateMultiMin, RateMultiMax.
1782
See DynamicRateLimitingParams comments for more details.`,
1783
)
1784
EnableBestEffortDeleteTasksOnWorkflowUpdate = NewGlobalBoolSetting(
1785
"history.enableBestEffortDeleteTasksOnWorkflowUpdate",
1786
false,
1787
`Enable deletion of requested history tasks (e.g., WFT timeout tasks) right after a successful UpdateWorkflowExecution.
1788
WARNING: Turning on this config can create a large number of tombstones in cassandra and degrade performance, use with caution.`,
1789
)
1790
EnableWorkflowTaskCompletionPagination = NewNamespaceBoolSetting(
1791
"history.enableWorkflowTaskCompletionPagination",
1792
false,
1793
`EnableWorkflowTaskCompletionPagination enables the pagination of RespondWorkflowTaskCompleted requests.
1794
When false, paginated requests (the ones with intermediate_page set to true) are rejected.`,
1795
)
1796
WorkflowTaskCompletionBufferSizeLimit = NewNamespaceIntSetting(
1797
"history.workflowTaskCompletionBufferSizeLimit",
1798
40*1024*1024,
1799
`WorkflowTaskCompletionBufferSizeLimit is the limit in bytes on the total
1800
size of buffered pages in paginated RespondWorkflowTaskCompleted requests for a single workflow task.`,
1801
)
1802
HistoryLongPollExpirationInterval = NewNamespaceDurationSetting(
1803
"history.longPollExpirationInterval",
1804
time.Second*20,
1805
`HistoryLongPollExpirationInterval is the long poll expiration interval in the history service`,
1806
)
1807
HistoryCacheSizeBasedLimit = NewGlobalBoolSetting(
1808
"history.cacheSizeBasedLimit",
1809
false,
1810
`HistoryCacheSizeBasedLimit if true, size of the history cache will be limited by HistoryCacheMaxSizeBytes
1811
and HistoryCacheHostLevelMaxSizeBytes. Otherwise, entry count in the history cache will be limited by
1812
HistoryCacheMaxSize and HistoryCacheHostLevelMaxSize. Requires service restart to take effect.`,
1813
)
1814
HistoryCacheTTL = NewGlobalDurationSetting(
1815
"history.cacheTTL",
1816
time.Hour,
1817
`HistoryCacheTTL is TTL of history cache. Requires service restart to take effect.`,
1818
)
1819
HistoryCacheNonUserContextLockTimeout = NewGlobalDurationSetting(
1820
"history.cacheNonUserContextLockTimeout",
1821
500*time.Millisecond,
1822
`HistoryCacheNonUserContextLockTimeout controls how long non-user call (callerType != API or Operator)
1823
will wait on workflow lock acquisition. Requires service restart to take effect.`,
1824
)
1825
HistoryCacheHostLevelMaxSize = NewGlobalIntSetting(
1826
"history.hostLevelCacheMaxSize",
1827
128000,
1828
`HistoryCacheHostLevelMaxSize is the maximum number of entries in the host level history cache.
1829
Requires service restart to take effect.`,
1830
)
1831
HistoryCacheHostLevelMaxSizeBytes = NewGlobalIntSetting(
1832
"history.hostLevelCacheMaxSizeBytes",
1833
256000*4*1024,
1834
`HistoryCacheHostLevelMaxSizeBytes is the maximum size of the host level history cache. This is only used if
1835
HistoryCacheSizeBasedLimit is set to true. Requires service restart to take effect.`,
1836
)
1837
HistoryCacheBackgroundEvict = NewGlobalTypedSetting(
1838
"history.cacheBackgroundEvict",
1839
DefaultHistoryCacheBackgroundEvictSettings,
1840
`HistoryCacheBackgroundEvict configures background processing to purge expired entries from the history cache.
1841
Requires service restart to take effect.`,
1842
)
1843
EnableWorkflowExecutionTimeoutTimer = NewGlobalBoolSetting(
1844
"history.enableWorkflowExecutionTimeoutTimer",
1845
true,
1846
`EnableWorkflowExecutionTimeoutTimer controls whether to enable the new logic for generating a workflow execution
1847
timeout timer when execution timeout is specified when starting a workflow.`,
1848
)
1849
EnableUpdateWorkflowModeIgnoreCurrent = NewGlobalBoolSetting(
1850
"history.enableUpdateWorkflowModeIgnoreCurrent",
1851
true,
1852
`EnableUpdateWorkflowModeIgnoreCurrent controls whether to enable the new logic for updating closed workflow execution
1853
by mutation using UpdateWorkflowModeIgnoreCurrent`,
1854
)
1855
EnableTransitionHistory = NewNamespaceBoolSetting(
1856
"history.enableTransitionHistory",
1857
true,
1858
`EnableTransitionHistory controls whether to enable the new logic for recording the history for each state transition.`,
1859
)
1860
HistoryStartupMembershipJoinDelay = NewGlobalDurationSetting(
1861
"history.startupMembershipJoinDelay",
1862
0*time.Second,
1863
`HistoryStartupMembershipJoinDelay is the duration a history instance waits
1864
before joining membership after starting.`,
1865
)
1866
HistoryAlignMembershipChange = NewGlobalDurationSetting(
1867
"history.alignMembershipChange",
1868
0*time.Second,
1869
`HistoryAlignMembershipChange is a duration to align history's membership changes to.
1870
This can help reduce effects of shard movement.`,
1871
)
1872
HistoryShutdownDrainDuration = NewGlobalDurationSetting(
1873
"history.shutdownDrainDuration",
1874
0*time.Second,
1875
`HistoryShutdownDrainDuration is the duration of traffic drain during shutdown`,
1876
)
1877
XDCCacheMaxSizeBytes = NewGlobalIntSetting(
1878
"history.xdcCacheMaxSizeBytes",
1879
8*1024*1024,
1880
`XDCCacheMaxSizeBytes is max size of events cache in bytes`,
1881
)
1882
EventsCacheMaxSizeBytes = NewGlobalIntSetting(
1883
"history.eventsCacheMaxSizeBytes",
1884
512*1024,
1885
`EventsCacheMaxSizeBytes is max size of the shard level events cache in bytes. Requires service restart to take effect.`,
1886
)
1887
EventsHostLevelCacheMaxSizeBytes = NewGlobalIntSetting(
1888
"history.eventsHostLevelCacheMaxSizeBytes",
1889
512*512*1024,
1890
`EventsHostLevelCacheMaxSizeBytes is max size of the host level events cache in bytes. Requires service restart to take effect.`,
1891
)
1892
EventsCacheTTL = NewGlobalDurationSetting(
1893
"history.eventsCacheTTL",
1894
time.Hour,
1895
`EventsCacheTTL is TTL of events cache. Requires service restart to take effect.`,
1896
)
1897
EnableHostLevelEventsCache = NewGlobalBoolSetting(
1898
"history.enableHostLevelEventsCache",
1899
false,
1900
`EnableHostLevelEventsCache controls if the events cache is host level. Requires service restart to take effect.`,
1901
)
1902
AcquireShardInterval = NewGlobalDurationSetting(
1903
"history.acquireShardInterval",
1904
time.Minute,
1905
`AcquireShardInterval is interval that timer used to acquire shard`,
1906
)
1907
AcquireShardConcurrency = NewGlobalIntSetting(
1908
"history.acquireShardConcurrency",
1909
10,
1910
`AcquireShardConcurrency is number of goroutines that can be used to acquire shards in the shard controller.`,
1911
)
1912
ShardLingerOwnershipCheckQPS = NewGlobalIntSetting(
1913
"history.shardLingerOwnershipCheckQPS",
1914
4,
1915
`ShardLingerOwnershipCheckQPS is the frequency to perform shard ownership
1916
checks while a shard is lingering.`,
1917
)
1918
ShardLingerTimeLimit = NewGlobalDurationSetting(
1919
"history.shardLingerTimeLimit",
1920
0,
1921
`ShardLingerTimeLimit configures if and for how long the shard controller
1922
will temporarily delay closing shards after a membership update, awaiting a
1923
shard ownership lost error from persistence. If set to zero, shards will not delay closing.
1924
Do NOT use non-zero value with persistence layers that are missing AssertShardOwnership support.`,
1925
)
1926
ShardFinalizerTimeout = NewGlobalDurationSetting(
1927
"history.shardFinalizerTimeout",
1928
2*time.Second,
1929
`ShardFinalizerTimeout configures if and for how long the shard will attempt
1930
to cleanup any of its associated data, such as workflow contexts. If set to zero, the finalizer is disabled.`,
1931
)
1932
HistoryClientOwnershipCachingEnabled = NewGlobalBoolSetting(
1933
"history.clientOwnershipCachingEnabled",
1934
false,
1935
`HistoryClientOwnershipCachingEnabled configures if history clients try to cache
1936
shard ownership information, instead of checking membership for each request.
1937
Only inspected when an instance first creates a history client, so changes
1938
to this require a restart to take effect.`,
1939
)
1940
HistoryClientOwnershipCachingStaleTTL = NewGlobalDurationSetting(
1941
"history.clientOwnershipCachingUnusedTTL",
1942
30*time.Second,
1943
`HistoryClientOwnershipCachingStaleTTL, if non-zero, configures the TTL
1944
for cached shard ownership entries after a membership update.
1945
Should be less than history.connectionCloseDelay so that connections are not
1946
closed while still cached.`,
1947
)
1948
HistoryConnectionCloseDelay = NewGlobalDurationSetting(
1949
"history.connectionCloseDelay",
1950
60*time.Second,
1951
`HistoryConnectionCloseDelay delays closing a cached history connection after its host leaves
1952
the membership ring, giving in-flight RPCs time to drain before the connection is closed.
1953
Should be greater than history.clientOwnershipCachingUnusedTTL so that connections are not closed
1954
while still cached.`,
1955
)
1956
ShardIOConcurrency = NewGlobalIntSetting(
1957
"history.shardIOConcurrency",
1958
1,
1959
`ShardIOConcurrency controls the concurrency of persistence operations in shard context`,
1960
)
1961
ShardIOTimeout = NewGlobalDurationSetting(
1962
"history.shardIOTimeout",
1963
5*time.Second*debug.TimeoutMultiplier,
1964
`ShardIOTimeout sets the timeout for persistence operations in the shard context`,
1965
)
1966
StandbyClusterDelay = NewGlobalDurationSetting(
1967
"history.standbyClusterDelay",
1968
5*time.Minute,
1969
`StandbyClusterDelay is the artificial delay added to standby cluster's view of active cluster's time`,
1970
)
1971
StandbyTaskMissingEventsResendDelay = NewTaskTypeDurationSetting(
1972
"history.standbyTaskMissingEventsResendDelay",
1973
10*time.Minute,
1974
`StandbyTaskMissingEventsResendDelay is the amount of time standby cluster's will wait (if events are missing)
1975
before calling remote for missing events`,
1976
)
1977
StandbyTaskMissingEventsDiscardDelay = NewTaskTypeDurationSetting(
1978
"history.standbyTaskMissingEventsDiscardDelay",
1979
15*time.Minute,
1980
`StandbyTaskMissingEventsDiscardDelay is the amount of time standby cluster's will wait (if events are missing)
1981
before discarding the task`,
1982
)
1983
ChasmStandbyTaskDiscardDelay = NewChasmTaskTypeDurationSetting(
1984
"history.ChasmStandbyTaskDiscardDelay",
1985
24*time.Hour,
1986
`ChasmStandbyTaskDiscardDelay is the amount of time standby cluster will wait
1987
before discarding a CHASM task. Configurable per RegistrableTask type (e.g. "activity.dispatch").
1988
The default is intentionally much higher than the non CHASM standby discard delay because
1989
discarding a CHASM task can leave the execution in a stuck state after failover. Task types
1990
that can be safely offloaded should be configured with a shorter delay.`,
1991
)
1992
QueuePendingTaskCriticalCount = NewGlobalIntSetting(
1993
"history.queuePendingTaskCriticalCount",
1994
9000,
1995
`Max number of pending tasks in a history queue before triggering slice splitting and unloading.
1996
NOTE: The outbound queue has a separate configuration: outboundQueuePendingTaskCriticalCount.`,
1997
)
1998
QueueReaderStuckCriticalAttempts = NewGlobalIntSetting(
1999
"history.queueReaderStuckCriticalAttempts",
2000
3,
2001
`QueueReaderStuckCriticalAttempts is the max number of task loading attempts for a certain task range
2002
before that task range is split into a separate slice to unblock loading for later range.
2003
currently only work for scheduled queues and the task range is 1s.`,
2004
)
2005
QueueCriticalSlicesCount = NewGlobalIntSetting(
2006
"history.queueCriticalSlicesCount",
2007
50,
2008
`QueueCriticalSlicesCount is the max number of slices in one queue
2009
before force compacting slices`,
2010
)
2011
QueuePendingTaskMaxCount = NewGlobalIntSetting(
2012
"history.queuePendingTasksMaxCount",
2013
10000,
2014
`The max number of task pending tasks in a history queue before stopping loading new tasks into memory. This
2015
limit is in addition to queuePendingTaskCriticalCount which controls when to unload already loaded tasks but doesn't
2016
prevent loading new tasks. Ideally this max count limit should not be hit and task unloading should happen once critical
2017
count is exceeded. But since queue action is async, we need this hard limit.
2018
NOTE: The outbound queue has a separate configuration: outboundQueuePendingTaskMaxCount.
2019
`,
2020
)
2021
QueueMaxPredicateSize = NewGlobalIntSetting(
2022
"history.queueMaxPredicateSize",
2023
10*1024,
2024
`The max size of the multi-cursor predicate structure stored in the shard info record. 0 is considered
2025
unlimited. When the predicate size is surpassed for a given scope, the predicate is converted to a universal predicate,
2026
which causes all tasks in the scope's range to eventually be reprocessed without applying any filtering logic.
2027
NOTE: The outbound queue has a separate configuration: outboundQueueMaxPredicateSize.
2028
`,
2029
)
2030
QueueShrinkPredicateMaxPendingKeys = NewGlobalIntSetting(
2031
"history.queueShrinkPredicateMaxPendingKeys",
2032
10,
2033
`Max number of pending task keys for which a multi-cursor slice shrinks its predicate back to exactly those
2034
keys.`,
2035
)
2036
QueueMoveGroupTaskCountBase = NewGlobalIntSetting(
2037
"history.queueMoveGroupTaskCountBase",
2038
500,
2039
`The base number of pending tasks count for a task group to be moved to the next level reader.
2040
The actual count is calculated as base * (multiplier ^ level)`,
2041
)
2042
QueueMoveGroupTaskCountMultiplier = NewGlobalFloatSetting(
2043
"history.queueMoveGroupTaskCountMultiplier",
2044
3.0,
2045
`The multiplier used to calculate the number of pending tasks for a task group to be moved to the next level reader.
2046
The actual count is calculated as base * (multiplier ^ level)`,
2047
)
2048
2049
TaskSchedulerEnableRateLimiter = NewGlobalBoolSetting(
2050
"history.taskSchedulerEnableRateLimiter",
2051
false,
2052
`TaskSchedulerEnableRateLimiter indicates if task scheduler rate limiter should be enabled`,
2053
)
2054
TaskSchedulerEnableRateLimiterShadowMode = NewGlobalBoolSetting(
2055
"history.taskSchedulerEnableRateLimiterShadowMode",
2056
true,
2057
`TaskSchedulerEnableRateLimiterShadowMode indicates if task scheduler rate limiter should run in shadow mode
2058
i.e. through rate limiter and emit metrics but do not actually block/throttle task scheduling`,
2059
)
2060
TaskSchedulerRateLimiterStartupDelay = NewGlobalDurationSetting(
2061
"history.taskSchedulerRateLimiterStartupDelay",
2062
5*time.Second,
2063
`TaskSchedulerRateLimiterStartupDelay is the duration to wait after startup before enforcing task scheduler rate limiting`,
2064
)
2065
TaskSchedulerGlobalMaxQPS = NewGlobalIntSetting(
2066
"history.taskSchedulerGlobalMaxQPS",
2067
0,
2068
`TaskSchedulerGlobalMaxQPS is the max qps all task schedulers in the cluster can schedule tasks
2069
If value less or equal to 0, will fall back to TaskSchedulerMaxQPS`,
2070
)
2071
TaskSchedulerMaxQPS = NewGlobalIntSetting(
2072
"history.taskSchedulerMaxQPS",
2073
0,
2074
`TaskSchedulerMaxQPS is the max qps task schedulers on a host can schedule tasks
2075
If value less or equal to 0, will fall back to HistoryPersistenceMaxQPS`,
2076
)
2077
TaskSchedulerGlobalNamespaceMaxQPS = NewNamespaceIntSetting(
2078
"history.taskSchedulerGlobalNamespaceMaxQPS",
2079
0,
2080
`TaskSchedulerGlobalNamespaceMaxQPS is the max qps all task schedulers in the cluster can schedule tasks for a certain namespace
2081
If value less or equal to 0, will fall back to TaskSchedulerNamespaceMaxQPS`,
2082
)
2083
TaskSchedulerNamespaceMaxQPS = NewNamespaceIntSetting(
2084
"history.taskSchedulerNamespaceMaxQPS",
2085
0,
2086
`TaskSchedulerNamespaceMaxQPS is the max qps task schedulers on a host can schedule tasks for a certain namespace
2087
If value less or equal to 0, will fall back to HistoryPersistenceNamespaceMaxQPS`,
2088
)
2089
TaskSchedulerInactiveChannelDeletionDelay = NewGlobalDurationSetting(
2090
"history.taskSchedulerInactiveChannelDeletionDelay",
2091
time.Hour,
2092
`TaskSchedulerInactiveChannelDeletionDelay the time delay before a namespace's' channel is removed from the scheduler`,
2093
)
2094
TaskSchedulerEnableExecutionQueueScheduler = NewGlobalBoolSetting(
2095
"history.taskSchedulerEnableExecutionQueueScheduler",
2096
false,
2097
`TaskSchedulerEnableExecutionQueueScheduler enables the execution queue scheduler
2098
that processes tasks for contended workflows sequentially to avoid busy workflow errors`,
2099
)
2100
TaskSchedulerExecutionQueueSchedulerMaxQueues = NewGlobalIntSetting(
2101
"history.taskSchedulerExecutionQueueSchedulerMaxQueues",
2102
500,
2103
`TaskSchedulerExecutionQueueSchedulerMaxQueues is the maximum number of concurrent per-workflow queues in the execution queue scheduler.
2104
When this limit is reached, new workflows will fall back to the base FIFO scheduler.`,
2105
)
2106
TaskSchedulerExecutionQueueSchedulerQueueTTL = NewGlobalDurationSetting(
2107
"history.taskSchedulerExecutionQueueSchedulerQueueTTL",
2108
5*time.Second,
2109
`TaskSchedulerExecutionQueueSchedulerQueueTTL is how long a per-workflow queue goroutine waits idle before exiting.`,
2110
)
2111
2112
TaskSchedulerExecutionQueueSchedulerQueueConcurrency = NewGlobalIntSetting(
2113
"history.taskSchedulerExecutionQueueSchedulerQueueConcurrency",
2114
2,
2115
`TaskSchedulerExecutionQueueSchedulerQueueConcurrency is the max number of worker goroutines per workflow queue.
2116
Higher values allow limited parallelism per workflow. Values <= 0 are capped to 1.`,
2117
)
2118
2119
TimerTaskBatchSize = NewGlobalIntSetting(
2120
"history.timerTaskBatchSize",
2121
100,
2122
`TimerTaskBatchSize is batch size for timer processor to process tasks`,
2123
)
2124
TimerProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2125
"history.timerProcessorSchedulerWorkerCount",
2126
512,
2127
`TimerProcessorSchedulerWorkerCount is the number of workers in the host level task scheduler for timer processor`,
2128
)
2129
TimerProcessorSchedulerActiveRoundRobinWeights = NewNamespaceMapSetting(
2130
"history.timerProcessorSchedulerActiveRoundRobinWeights",
2131
nil, // actual default is in service/history/configs package
2132
`TimerProcessorSchedulerActiveRoundRobinWeights is the priority round robin weights used by timer task scheduler for active namespaces`,
2133
)
2134
TimerProcessorSchedulerStandbyRoundRobinWeights = NewNamespaceMapSetting(
2135
"history.timerProcessorSchedulerStandbyRoundRobinWeights",
2136
nil, // actual default is in service/history/configs package
2137
`TimerProcessorSchedulerStandbyRoundRobinWeights is the priority round robin weights used by timer task scheduler for standby namespaces`,
2138
)
2139
TimerProcessorUpdateAckInterval = NewGlobalDurationSetting(
2140
"history.timerProcessorUpdateAckInterval",
2141
30*time.Second,
2142
`TimerProcessorUpdateAckInterval is update interval for timer processor`,
2143
)
2144
TimerProcessorUpdateAckIntervalJitterCoefficient = NewGlobalFloatSetting(
2145
"history.timerProcessorUpdateAckIntervalJitterCoefficient",
2146
0.15,
2147
`TimerProcessorUpdateAckIntervalJitterCoefficient is the update interval jitter coefficient`,
2148
)
2149
TimerProcessorMaxPollRPS = NewGlobalIntSetting(
2150
"history.timerProcessorMaxPollRPS",
2151
20,
2152
`TimerProcessorMaxPollRPS is max poll rate per second for timer processor`,
2153
)
2154
TimerProcessorMaxPollHostRPS = NewGlobalIntSetting(
2155
"history.timerProcessorMaxPollHostRPS",
2156
0,
2157
`TimerProcessorMaxPollHostRPS is max poll rate per second for all timer processor on a host`,
2158
)
2159
TimerProcessorMaxPollInterval = NewGlobalDurationSetting(
2160
"history.timerProcessorMaxPollInterval",
2161
5*time.Minute,
2162
`TimerProcessorMaxPollInterval is max poll interval for timer processor`,
2163
)
2164
TimerProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2165
"history.timerProcessorMaxPollIntervalJitterCoefficient",
2166
0.15,
2167
`TimerProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2168
)
2169
TimerProcessorPollBackoffInterval = NewGlobalDurationSetting(
2170
"history.timerProcessorPollBackoffInterval",
2171
5*time.Second,
2172
`TimerProcessorPollBackoffInterval is the poll backoff interval if task redispatcher's size exceeds limit for timer processor`,
2173
)
2174
TimerProcessorMaxTimeShift = NewGlobalDurationSetting(
2175
"history.timerProcessorMaxTimeShift",
2176
1*time.Second,
2177
`TimerProcessorMaxTimeShift is the max shift timer processor can have`,
2178
)
2179
TimerQueueMaxReaderCount = NewGlobalIntSetting(
2180
"history.timerQueueMaxReaderCount",
2181
2,
2182
`TimerQueueMaxReaderCount is the max number of readers in one multi-cursor timer queue`,
2183
)
2184
RetentionTimerJitterDuration = NewGlobalDurationSetting(
2185
"history.retentionTimerJitterDuration",
2186
30*time.Minute,
2187
`RetentionTimerJitterDuration is a time duration jitter to distribute timer from T0 to T0 + jitter duration`,
2188
)
2189
2190
MemoryTimerProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2191
"history.memoryTimerProcessorSchedulerWorkerCount",
2192
64,
2193
`MemoryTimerProcessorSchedulerWorkerCount is the number of workers in the task scheduler for in memory timer processor.`,
2194
)
2195
2196
TransferTaskBatchSize = NewGlobalIntSetting(
2197
"history.transferTaskBatchSize",
2198
100,
2199
`TransferTaskBatchSize is batch size for transferQueueProcessor`,
2200
)
2201
TransferProcessorMaxPollRPS = NewGlobalIntSetting(
2202
"history.transferProcessorMaxPollRPS",
2203
20,
2204
`TransferProcessorMaxPollRPS is max poll rate per second for transferQueueProcessor`,
2205
)
2206
TransferProcessorMaxPollHostRPS = NewGlobalIntSetting(
2207
"history.transferProcessorMaxPollHostRPS",
2208
0,
2209
`TransferProcessorMaxPollHostRPS is max poll rate per second for all transferQueueProcessor on a host`,
2210
)
2211
TransferProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2212
"history.transferProcessorSchedulerWorkerCount",
2213
512,
2214
`TransferProcessorSchedulerWorkerCount is the number of workers in the host level task scheduler for transferQueueProcessor`,
2215
)
2216
TransferProcessorSchedulerActiveRoundRobinWeights = NewNamespaceMapSetting(
2217
"history.transferProcessorSchedulerActiveRoundRobinWeights",
2218
nil, // actual default is in service/history/configs package
2219
`TransferProcessorSchedulerActiveRoundRobinWeights is the priority round robin weights used by transfer task scheduler for active namespaces`,
2220
)
2221
TransferProcessorSchedulerStandbyRoundRobinWeights = NewNamespaceMapSetting(
2222
"history.transferProcessorSchedulerStandbyRoundRobinWeights",
2223
nil, // actual default is in service/history/configs package
2224
`TransferProcessorSchedulerStandbyRoundRobinWeights is the priority round robin weights used by transfer task scheduler for standby namespaces`,
2225
)
2226
TransferProcessorMaxPollInterval = NewGlobalDurationSetting(
2227
"history.transferProcessorMaxPollInterval",
2228
1*time.Minute,
2229
`TransferProcessorMaxPollInterval max poll interval for transferQueueProcessor`,
2230
)
2231
TransferProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2232
"history.transferProcessorMaxPollIntervalJitterCoefficient",
2233
0.15,
2234
`TransferProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2235
)
2236
TransferProcessorUpdateAckInterval = NewGlobalDurationSetting(
2237
"history.transferProcessorUpdateAckInterval",
2238
30*time.Second,
2239
`TransferProcessorUpdateAckInterval is update interval for transferQueueProcessor`,
2240
)
2241
TransferProcessorUpdateAckIntervalJitterCoefficient = NewGlobalFloatSetting(
2242
"history.transferProcessorUpdateAckIntervalJitterCoefficient",
2243
0.15,
2244
`TransferProcessorUpdateAckIntervalJitterCoefficient is the update interval jitter coefficient`,
2245
)
2246
TransferProcessorPollBackoffInterval = NewGlobalDurationSetting(
2247
"history.transferProcessorPollBackoffInterval",
2248
5*time.Second,
2249
`TransferProcessorPollBackoffInterval is the poll backoff interval if task redispatcher's size exceeds limit for transferQueueProcessor`,
2250
)
2251
TransferProcessorEnsureCloseBeforeDelete = NewGlobalBoolSetting(
2252
"history.transferProcessorEnsureCloseBeforeDelete",
2253
true,
2254
`TransferProcessorEnsureCloseBeforeDelete means we ensure the execution is closed before we delete it`,
2255
)
2256
TransferQueueMaxReaderCount = NewGlobalIntSetting(
2257
"history.transferQueueMaxReaderCount",
2258
2,
2259
`TransferQueueMaxReaderCount is the max number of readers in one multi-cursor transfer queue`,
2260
)
2261
2262
OutboundTaskBatchSize = NewGlobalIntSetting(
2263
"history.outboundTaskBatchSize",
2264
100,
2265
`OutboundTaskBatchSize is batch size for outboundQueueFactory`,
2266
)
2267
OutboundQueuePendingTaskMaxCount = NewGlobalIntSetting(
2268
"history.outboundQueuePendingTasksMaxCount",
2269
10000,
2270
`The max number of task pending tasks in the outbound queue before stopping loading new tasks into memory. This
2271
limit is in addition to outboundQueuePendingTaskCriticalCount which controls when to unload already loaded tasks but
2272
doesn't prevent loading new tasks. Ideally this max count limit should not be hit and task unloading should happen once
2273
critical count is exceeded. But since queue action is async, we need this hard limit.
2274
`,
2275
)
2276
OutboundQueuePendingTaskCriticalCount = NewGlobalIntSetting(
2277
"history.outboundQueuePendingTaskCriticalCount",
2278
9000,
2279
`Max number of pending tasks in the outbound queue before triggering slice splitting and unloading.`,
2280
)
2281
OutboundQueueMaxPredicateSize = NewGlobalIntSetting(
2282
"history.outboundQueueMaxPredicateSize",
2283
10*1024,
2284
`The max size of the multi-cursor predicate structure stored in the shard info record for the outbound queue. 0
2285
is considered unlimited. When the predicate size is surpassed for a given scope, the predicate is converted to a
2286
universal predicate, which causes all tasks in the scope's range to eventually be reprocessed without applying any
2287
filtering logic.
2288
`,
2289
)
2290
2291
OutboundProcessorMaxPollRPS = NewGlobalIntSetting(
2292
"history.outboundProcessorMaxPollRPS",
2293
20,
2294
`OutboundProcessorMaxPollRPS is max poll rate per second for outboundQueueFactory`,
2295
)
2296
OutboundProcessorMaxPollHostRPS = NewGlobalIntSetting(
2297
"history.outboundProcessorMaxPollHostRPS",
2298
0,
2299
`OutboundProcessorMaxPollHostRPS is max poll rate per second for all outboundQueueFactory on a host`,
2300
)
2301
OutboundProcessorMaxPollInterval = NewGlobalDurationSetting(
2302
"history.outboundProcessorMaxPollInterval",
2303
1*time.Minute,
2304
`OutboundProcessorMaxPollInterval max poll interval for outboundQueueFactory`,
2305
)
2306
OutboundProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2307
"history.outboundProcessorMaxPollIntervalJitterCoefficient",
2308
0.15,
2309
`OutboundProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2310
)
2311
OutboundProcessorUpdateAckInterval = NewGlobalDurationSetting(
2312
"history.outboundProcessorUpdateAckInterval",
2313
30*time.Second,
2314
`OutboundProcessorUpdateAckInterval is update interval for outboundQueueFactory`,
2315
)
2316
OutboundProcessorUpdateAckIntervalJitterCoefficient = NewGlobalFloatSetting(
2317
"history.outboundProcessorUpdateAckIntervalJitterCoefficient",
2318
0.15,
2319
`OutboundProcessorUpdateAckIntervalJitterCoefficient is the update interval jitter coefficient`,
2320
)
2321
OutboundProcessorPollBackoffInterval = NewGlobalDurationSetting(
2322
"history.outboundProcessorPollBackoffInterval",
2323
5*time.Second,
2324
`OutboundProcessorPollBackoffInterval is the poll backoff interval if task redispatcher's size exceeds limit for outboundQueueFactory`,
2325
)
2326
OutboundQueueMaxReaderCount = NewGlobalIntSetting(
2327
"history.outboundQueueMaxReaderCount",
2328
4,
2329
`OutboundQueueMaxReaderCount is the max number of readers in one multi-cursor outbound queue`,
2330
)
2331
OutboundQueueGroupLimiterBufferSize = NewDestinationIntSetting(
2332
"history.outboundQueue.groupLimiter.bufferSize",
2333
100,
2334
`OutboundQueueGroupLimiterBufferSize is the max buffer size of the group limiter`,
2335
)
2336
OutboundQueueGroupLimiterConcurrency = NewDestinationIntSetting(
2337
"history.outboundQueue.groupLimiter.concurrency",
2338
100,
2339
`OutboundQueueGroupLimiterConcurrency is the concurrency of the group limiter`,
2340
)
2341
OutboundQueueHostSchedulerMaxTaskRPS = NewDestinationFloatSetting(
2342
"history.outboundQueue.hostScheduler.maxTaskRPS",
2343
100.0,
2344
`OutboundQueueHostSchedulerMaxTaskRPS is the host scheduler max task RPS`,
2345
)
2346
OutboundQueueCircuitBreakerSettings = NewDestinationTypedSetting(
2347
"history.outboundQueue.circuitBreakerSettings",
2348
CircuitBreakerSettings{},
2349
`OutboundQueueCircuitBreakerSettings are circuit breaker settings.
2350
Fields (see gobreaker reference for more details):
2351
- MaxRequests: Maximum number of requests allowed to pass through when it is half-open (default 1).
2352
- Interval (duration): Cyclic period in closed state to clear the internal counts;
2353
if interval is 0, then it never clears the internal counts (default 0).
2354
- Timeout (duration): Period of open state before changing to half-open state (default 60s).`,
2355
)
2356
OutboundStandbyTaskMissingEventsDiscardDelay = NewDestinationDurationSetting(
2357
"history.outboundQueue.standbyTaskMissingEventsDiscardDelay",
2358
// This is effectively equivalent to never discarding outbound tasks since it's 290+ years.
2359
time.Duration(math.MaxInt64),
2360
`OutboundStandbyTaskMissingEventsDiscardDelay is the equivalent of
2361
StandbyTaskMissingEventsDiscardDelay for outbound standby task processor.`,
2362
)
2363
OutboundStandbyTaskMissingEventsDestinationDownErr = NewDestinationBoolSetting(
2364
"history.outboundQueue.standbyTaskMissingEventsDestinationDownErr",
2365
true,
2366
`OutboundStandbyTaskMissingEventsDestinationDownErr enables returning DestinationDownError when
2367
the outbound standby task failed to be processed due to missing events.`,
2368
)
2369
2370
VisibilityTaskBatchSize = NewGlobalIntSetting(
2371
"history.visibilityTaskBatchSize",
2372
100,
2373
`VisibilityTaskBatchSize is batch size for visibilityQueueProcessor`,
2374
)
2375
VisibilityProcessorMaxPollRPS = NewGlobalIntSetting(
2376
"history.visibilityProcessorMaxPollRPS",
2377
20,
2378
`VisibilityProcessorMaxPollRPS is max poll rate per second for visibilityQueueProcessor`,
2379
)
2380
VisibilityProcessorMaxPollHostRPS = NewGlobalIntSetting(
2381
"history.visibilityProcessorMaxPollHostRPS",
2382
0,
2383
`VisibilityProcessorMaxPollHostRPS is max poll rate per second for all visibilityQueueProcessor on a host`,
2384
)
2385
VisibilityProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2386
"history.visibilityProcessorSchedulerWorkerCount",
2387
512,
2388
`VisibilityProcessorSchedulerWorkerCount is the number of workers in the host level task scheduler for visibilityQueueProcessor`,
2389
)
2390
VisibilityProcessorSchedulerActiveRoundRobinWeights = NewNamespaceMapSetting(
2391
"history.visibilityProcessorSchedulerActiveRoundRobinWeights",
2392
nil, // actual default is in service/history/configs package
2393
`VisibilityProcessorSchedulerActiveRoundRobinWeights is the priority round robin weights by visibility task scheduler for active namespaces`,
2394
)
2395
VisibilityProcessorSchedulerStandbyRoundRobinWeights = NewNamespaceMapSetting(
2396
"history.visibilityProcessorSchedulerStandbyRoundRobinWeights",
2397
nil, // actual default is in service/history/configs package
2398
`VisibilityProcessorSchedulerStandbyRoundRobinWeights is the priority round robin weights by visibility task scheduler for standby namespaces`,
2399
)
2400
VisibilityProcessorMaxPollInterval = NewGlobalDurationSetting(
2401
"history.visibilityProcessorMaxPollInterval",
2402
1*time.Minute,
2403
`VisibilityProcessorMaxPollInterval max poll interval for visibilityQueueProcessor`,
2404
)
2405
VisibilityProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2406
"history.visibilityProcessorMaxPollIntervalJitterCoefficient",
2407
0.15,
2408
`VisibilityProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2409
)
2410
VisibilityProcessorUpdateAckInterval = NewGlobalDurationSetting(
2411
"history.visibilityProcessorUpdateAckInterval",
2412
30*time.Second,
2413
`VisibilityProcessorUpdateAckInterval is update interval for visibilityQueueProcessor`,
2414
)
2415
VisibilityProcessorUpdateAckIntervalJitterCoefficient = NewGlobalFloatSetting(
2416
"history.visibilityProcessorUpdateAckIntervalJitterCoefficient",
2417
0.15,
2418
`VisibilityProcessorUpdateAckIntervalJitterCoefficient is the update interval jitter coefficient`,
2419
)
2420
VisibilityProcessorPollBackoffInterval = NewGlobalDurationSetting(
2421
"history.visibilityProcessorPollBackoffInterval",
2422
5*time.Second,
2423
`VisibilityProcessorPollBackoffInterval is the poll backoff interval if task redispatcher's size exceeds limit for visibilityQueueProcessor`,
2424
)
2425
VisibilityProcessorEnsureCloseBeforeDelete = NewGlobalBoolSetting(
2426
"history.visibilityProcessorEnsureCloseBeforeDelete",
2427
false,
2428
`VisibilityProcessorEnsureCloseBeforeDelete means we ensure the visibility of an execution is closed before we delete its visibility records`,
2429
)
2430
VisibilityProcessorEnableCloseWorkflowCleanup = NewNamespaceBoolSetting(
2431
"history.visibilityProcessorEnableCloseWorkflowCleanup",
2432
false,
2433
`VisibilityProcessorEnableCloseWorkflowCleanup to clean up the mutable state after visibility
2434
close task has been processed. Must use Elasticsearch as visibility store, otherwise workflow
2435
data (eg: search attributes) will be lost after workflow is closed.`,
2436
)
2437
VisibilityProcessorRelocateAttributesMinBlobSize = NewNamespaceIntSetting(
2438
"history.visibilityProcessorRelocateAttributesMinBlobSize",
2439
0,
2440
`VisibilityProcessorRelocateAttributesMinBlobSize is the minimum size in bytes of memo or search
2441
attributes.`,
2442
)
2443
VisibilityQueueMaxReaderCount = NewGlobalIntSetting(
2444
"history.visibilityQueueMaxReaderCount",
2445
2,
2446
`VisibilityQueueMaxReaderCount is the max number of readers in one multi-cursor visibility queue`,
2447
)
2448
2449
DisableFetchRelocatableAttributesFromVisibility = NewNamespaceBoolSetting(
2450
"history.disableFetchRelocatableAttributesFromVisibility",
2451
false,
2452
`DisableFetchRelocatableAttributesFromVisibility disables fetching memo and search attributes from
2453
visibility if they were removed from the mutable state`,
2454
)
2455
2456
ArchivalTaskBatchSize = NewGlobalIntSetting(
2457
"history.archivalTaskBatchSize",
2458
100,
2459
`ArchivalTaskBatchSize is batch size for archivalQueueProcessor`,
2460
)
2461
ArchivalProcessorMaxPollRPS = NewGlobalIntSetting(
2462
"history.archivalProcessorMaxPollRPS",
2463
20,
2464
`ArchivalProcessorMaxPollRPS is max poll rate per second for archivalQueueProcessor`,
2465
)
2466
ArchivalProcessorMaxPollHostRPS = NewGlobalIntSetting(
2467
"history.archivalProcessorMaxPollHostRPS",
2468
0,
2469
`ArchivalProcessorMaxPollHostRPS is max poll rate per second for all archivalQueueProcessor on a host`,
2470
)
2471
ArchivalProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2472
"history.archivalProcessorSchedulerWorkerCount",
2473
512,
2474
`ArchivalProcessorSchedulerWorkerCount is the number of workers in the host level task scheduler for
2475
archivalQueueProcessor`,
2476
)
2477
ArchivalProcessorMaxPollInterval = NewGlobalDurationSetting(
2478
"history.archivalProcessorMaxPollInterval",
2479
5*time.Minute,
2480
`ArchivalProcessorMaxPollInterval max poll interval for archivalQueueProcessor`,
2481
)
2482
ArchivalProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2483
"history.archivalProcessorMaxPollIntervalJitterCoefficient",
2484
0.15,
2485
`ArchivalProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2486
)
2487
ArchivalProcessorUpdateAckInterval = NewGlobalDurationSetting(
2488
"history.archivalProcessorUpdateAckInterval",
2489
30*time.Second,
2490
`ArchivalProcessorUpdateAckInterval is update interval for archivalQueueProcessor`,
2491
)
2492
ArchivalProcessorUpdateAckIntervalJitterCoefficient = NewGlobalFloatSetting(
2493
"history.archivalProcessorUpdateAckIntervalJitterCoefficient",
2494
0.15,
2495
`ArchivalProcessorUpdateAckIntervalJitterCoefficient is the update interval jitter coefficient`,
2496
)
2497
ArchivalProcessorPollBackoffInterval = NewGlobalDurationSetting(
2498
"history.archivalProcessorPollBackoffInterval",
2499
5*time.Second,
2500
`ArchivalProcessorPollBackoffInterval is the poll backoff interval if task redispatcher's size exceeds limit for
2501
archivalQueueProcessor`,
2502
)
2503
ArchivalProcessorArchiveDelay = NewGlobalDurationSetting(
2504
"history.archivalProcessorArchiveDelay",
2505
5*time.Minute,
2506
`ArchivalProcessorArchiveDelay is the delay before archivalQueueProcessor starts to process archival tasks`,
2507
)
2508
ArchivalBackendMaxRPS = NewGlobalFloatSetting(
2509
"history.archivalBackendMaxRPS",
2510
10000.0,
2511
`ArchivalBackendMaxRPS is the maximum rate of requests per second to the archival backend`,
2512
)
2513
ArchivalQueueMaxReaderCount = NewGlobalIntSetting(
2514
"history.archivalQueueMaxReaderCount",
2515
2,
2516
`ArchivalQueueMaxReaderCount is the max number of readers in one multi-cursor archival queue`,
2517
)
2518
2519
WorkflowExecutionMaxInFlightUpdates = NewNamespaceIntSetting(
2520
"history.maxInFlightUpdates",
2521
10,
2522
`WorkflowExecutionMaxInFlightUpdates is the max number of updates that can be in-flight (admitted but not yet completed) for any given workflow execution. Set to zero to disable limit.`,
2523
)
2524
WorkflowExecutionMaxInFlightUpdatePayloads = NewNamespaceIntSetting(
2525
"history.maxInFlightUpdatePayloads",
2526
20*1024*1024,
2527
`WorkflowExecutionMaxInFlightUpdatePayloads is the max total payload size (in bytes) of in-flight updates (admitted but not yet completed) for any given workflow execution. Set to zero to disable.`,
2528
)
2529
WorkflowExecutionMaxTotalUpdates = NewNamespaceIntSetting(
2530
"history.maxTotalUpdates",
2531
2000,
2532
`WorkflowExecutionMaxTotalUpdates is the max number of updates that any given workflow execution can receive. Set to zero to disable.`,
2533
)
2534
WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold = NewNamespaceFloatSetting(
2535
"history.maxTotalUpdates.suggestContinueAsNewThreshold",
2536
0.9,
2537
`WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold is the percentage threshold of total updates that any given workflow execution can receive before suggesting to continue-as-new.`,
2538
)
2539
EnableUpdateWithStartRetryOnClosedWorkflowAbort = NewNamespaceBoolSetting(
2540
"history.enableUpdateWithStartRetryOnClosedWorkflowAbort",
2541
true,
2542
`EnableUpdateWithStartRetryOnClosedWorkflowAbort enables retrying Update-with-Start's update if it was aborted by a closing workflow.`,
2543
)
2544
EnableUpdateWithStartRetryableErrorOnClosedWorkflowAbort = NewNamespaceBoolSetting(
2545
"history.enableUpdateWithStartRetryableErrorOnClosedWorkflowAbort",
2546
true,
2547
`EnableUpdateWithStartRetryableErrorOnClosedWorkflowAbort enables sending back a retryable status code when the Update-with-Start's update was aborted by a closing workflow.`,
2548
)
2549
2550
ReplicatorTaskBatchSize = NewGlobalIntSetting(
2551
"history.replicatorTaskBatchSize",
2552
100,
2553
`ReplicatorTaskBatchSize is batch size for ReplicatorProcessor`,
2554
)
2555
ReplicatorMaxSkipTaskCount = NewGlobalIntSetting(
2556
"history.replicatorMaxSkipTaskCount",
2557
250,
2558
`ReplicatorMaxSkipTaskCount is maximum number of tasks that can be skipped during tasks pagination due to not meeting filtering conditions (e.g. missed namespace).`,
2559
)
2560
ReplicatorProcessorMaxPollInterval = NewGlobalDurationSetting(
2561
"history.replicatorProcessorMaxPollInterval",
2562
1*time.Minute,
2563
`ReplicatorProcessorMaxPollInterval is max poll interval for ReplicatorProcessor`,
2564
)
2565
ReplicatorProcessorMaxPollIntervalJitterCoefficient = NewGlobalFloatSetting(
2566
"history.replicatorProcessorMaxPollIntervalJitterCoefficient",
2567
0.15,
2568
`ReplicatorProcessorMaxPollIntervalJitterCoefficient is the max poll interval jitter coefficient`,
2569
)
2570
MaximumBufferedEventsBatch = NewGlobalIntSetting(
2571
"history.maximumBufferedEventsBatch",
2572
100,
2573
`MaximumBufferedEventsBatch is the maximum permissible number of buffered events for any given mutable state.`,
2574
)
2575
MaximumBufferedEventsSizeInBytes = NewGlobalIntSetting(
2576
"history.maximumBufferedEventsSizeInBytes",
2577
2*1024*1024,
2578
`MaximumBufferedEventsSizeInBytes is the maximum permissible size of all buffered events for any given mutable
2579
state. The total size is determined by the sum of the size, in bytes, of each HistoryEvent proto.`,
2580
)
2581
MaximumEventBatchSizeInBytes = NewGlobalIntSetting(
2582
"history.maximumEventBatchSizeInBytes",
2583
0,
2584
`This is EXPERIMENTAL feature that is under development. Things can break if you use it.
2585
MaximumEventBatchSizeInBytes is the size threshold (in bytes) at which the EventStore rolls the
2586
current in-memory batch and starts a new one. A single oversized event may cause a batch to
2587
exceed this size. A value of 0 disables the check. This value should stay below
2588
system.transactionSizeLimit, since each batch is persisted within a single transaction.`,
2589
)
2590
MaximumSignalsPerExecution = NewNamespaceIntSetting(
2591
"history.maximumSignalsPerExecution",
2592
10000,
2593
`MaximumSignalsPerExecution is max number of signals supported by single execution`,
2594
)
2595
ShardUpdateMinInterval = NewGlobalDurationSetting(
2596
"history.shardUpdateMinInterval",
2597
5*time.Minute,
2598
`ShardUpdateMinInterval is the minimal time interval which the shard info can be updated`,
2599
)
2600
ShardFirstUpdateInterval = NewGlobalDurationSetting(
2601
"history.shardFirstUpdateInterval",
2602
10*time.Second,
2603
`ShardFirstUpdateInterval is the time interval after which the first shard info update will happen.
2604
It should be smaller than ShardUpdateMinInterval`,
2605
)
2606
ShardUpdateMinTasksCompleted = NewGlobalIntSetting(
2607
"history.shardUpdateMinTasksCompleted",
2608
1000,
2609
`ShardUpdateMinTasksCompleted is the minimum number of tasks which must be completed (across all queues) before the shard info can be updated.
2610
Note that once history.shardUpdateMinInterval amount of time has passed we'll update the shard info regardless of the number of tasks completed.
2611
When the this config is zero or lower we will only update shard info at most once every history.shardUpdateMinInterval.`,
2612
)
2613
ShardSyncMinInterval = NewGlobalDurationSetting(
2614
"history.shardSyncMinInterval",
2615
5*time.Minute,
2616
`ShardSyncMinInterval is the minimal time interval which the shard info should be sync to remote`,
2617
)
2618
EmitShardLagLog = NewGlobalBoolSetting(
2619
"history.emitShardLagLog",
2620
false,
2621
`EmitShardLagLog whether emit the shard lag log`,
2622
)
2623
DefaultActivityRetryPolicy = NewNamespaceTypedSetting(
2624
"history.defaultActivityRetryPolicy",
2625
retrypolicy.DefaultDefaultRetrySettings,
2626
`DefaultActivityRetryPolicy represents the out-of-box retry policy for activities where
2627
the user has not specified an explicit RetryPolicy`,
2628
)
2629
DefaultWorkflowRetryPolicy = NewNamespaceTypedSetting(
2630
"history.defaultWorkflowRetryPolicy",
2631
retrypolicy.DefaultDefaultRetrySettings,
2632
`DefaultWorkflowRetryPolicy represents the out-of-box retry policy for unset fields
2633
where the user has set an explicit RetryPolicy, but not specified all the fields`,
2634
)
2635
AllowResetWithPendingChildren = NewNamespaceBoolSetting(
2636
"history.allowResetWithPendingChildren",
2637
true,
2638
`Allows resetting of workflows with pending children when set to true`,
2639
)
2640
HistoryMaxAutoResetPoints = NewNamespaceIntSetting(
2641
"history.historyMaxAutoResetPoints",
2642
primitives.DefaultHistoryMaxAutoResetPoints,
2643
`HistoryMaxAutoResetPoints is the key for max number of auto reset points stored in mutableState`,
2644
)
2645
EnableParentClosePolicy = NewNamespaceBoolSetting(
2646
"history.enableParentClosePolicy",
2647
true,
2648
`EnableParentClosePolicy whether to ParentClosePolicy`,
2649
)
2650
ParentClosePolicyThreshold = NewNamespaceIntSetting(
2651
"history.parentClosePolicyThreshold",
2652
10,
2653
`ParentClosePolicyThreshold decides that parent close policy will be processed by sys workers(if enabled) if
2654
the number of children greater than or equal to this threshold`,
2655
)
2656
NumParentClosePolicySystemWorkflows = NewGlobalIntSetting(
2657
"history.numParentClosePolicySystemWorkflows",
2658
1000,
2659
`NumParentClosePolicySystemWorkflows is key for number of parentClosePolicy system workflows running in total`,
2660
)
2661
HistoryThrottledLogRPS = NewGlobalIntSetting(
2662
"history.throttledLogRPS",
2663
4,
2664
`HistoryThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger`,
2665
)
2666
WorkflowTaskHeartbeatTimeout = NewNamespaceDurationSetting(
2667
"history.workflowTaskHeartbeatTimeout",
2668
time.Minute*30,
2669
`WorkflowTaskHeartbeatTimeout for workflow task heartbeat`,
2670
)
2671
WorkflowTaskCriticalAttempts = NewGlobalIntSetting(
2672
"history.workflowTaskCriticalAttempt",
2673
10,
2674
`WorkflowTaskCriticalAttempts is the number of attempts for a workflow task that's regarded as critical`,
2675
)
2676
WorkflowTaskRetryMaxInterval = NewGlobalDurationSetting(
2677
"history.workflowTaskRetryMaxInterval",
2678
time.Minute*10,
2679
`WorkflowTaskRetryMaxInterval is the maximum interval added to a workflow task's startToClose timeout for slowing down retry`,
2680
)
2681
EnableWorkflowTaskStampIncrementOnFailure = NewGlobalBoolSetting(
2682
"history.enableWorkflowTaskStampIncrementOnFailure",
2683
false,
2684
`EnableWorkflowTaskStampIncrementOnFailure controls whether the workflow task stamp is incremented when a workflow task fails and is rescheduled`,
2685
)
2686
DiscardSpeculativeWorkflowTaskMaximumEventsCount = NewGlobalIntSetting(
2687
"history.discardSpeculativeWorkflowTaskMaximumEventsCount",
2688
10,
2689
`If speculative workflow task shipped more than DiscardSpeculativeWorkflowTaskMaximumEventsCount events, it can't be discarded`,
2690
)
2691
EnableDropRepeatedWorkflowTaskFailures = NewNamespaceBoolSetting(
2692
"history.enableDropRepeatedWorkflowTaskFailures",
2693
false,
2694
`EnableDropRepeatedWorkflowTaskFailures whether to silently drop repeated workflow task failures`,
2695
)
2696
SendTransientOrSpeculativeWorkflowTaskEvents = NewNamespaceBoolSetting(
2697
"history.sendTransientOrSpeculativeWorkflowTaskEvents",
2698
true,
2699
`SendTransientOrSpeculativeWorkflowTaskEvents controls whether GetWorkflowExecutionHistory returns non-durable transient or speculative workflow task events. Enabled by default but can be disabled per namespace if it causes compatibility problems.`,
2700
)
2701
DefaultWorkflowTaskTimeout = NewNamespaceDurationSetting(
2702
"history.defaultWorkflowTaskTimeout",
2703
primitives.DefaultWorkflowTaskTimeout,
2704
`DefaultWorkflowTaskTimeout for a workflow task`,
2705
)
2706
SkipReapplicationByNamespaceID = NewNamespaceIDBoolSetting(
2707
"history.SkipReapplicationByNamespaceID",
2708
false,
2709
`SkipReapplicationByNamespaceID is whether skipping a event re-application for a namespace`,
2710
)
2711
StandbyTaskReReplicationContextTimeout = NewNamespaceIDDurationSetting(
2712
"history.standbyTaskReReplicationContextTimeout",
2713
30*time.Second,
2714
`StandbyTaskReReplicationContextTimeout is the context timeout for standby task re-replication`,
2715
)
2716
MaxBufferedQueryCount = NewGlobalIntSetting(
2717
"history.MaxBufferedQueryCount",
2718
1,
2719
`MaxBufferedQueryCount indicates max buffer query count`,
2720
)
2721
MutableStateChecksumGenProbability = NewNamespaceIntSetting(
2722
"history.mutableStateChecksumGenProbability",
2723
0,
2724
`MutableStateChecksumGenProbability is the probability [0-100] that checksum will be generated for mutable state`,
2725
)
2726
MutableStateChecksumVerifyProbability = NewNamespaceIntSetting(
2727
"history.mutableStateChecksumVerifyProbability",
2728
0,
2729
`MutableStateChecksumVerifyProbability is the probability [0-100] that checksum will be verified for mutable state`,
2730
)
2731
MutableStateChecksumInvalidateBefore = NewGlobalFloatSetting(
2732
"history.mutableStateChecksumInvalidateBefore",
2733
0,
2734
`MutableStateChecksumInvalidateBefore is the epoch timestamp before which all checksums are to be discarded`,
2735
)
2736
2737
ReplicationTaskApplyTimeout = NewGlobalDurationSetting(
2738
"history.ReplicationTaskApplyTimeout",
2739
20*time.Second,
2740
`ReplicationTaskApplyTimeout is the context timeout for replication task apply`,
2741
)
2742
ReplicationTaskFetcherParallelism = NewGlobalIntSetting(
2743
"history.ReplicationTaskFetcherParallelism",
2744
4,
2745
`ReplicationTaskFetcherParallelism determines how many go routines we spin up for fetching tasks`,
2746
)
2747
ReplicationTaskFetcherAggregationInterval = NewGlobalDurationSetting(
2748
"history.ReplicationTaskFetcherAggregationInterval",
2749
2*time.Second,
2750
`ReplicationTaskFetcherAggregationInterval determines how frequently the fetch requests are sent`,
2751
)
2752
ReplicationTaskFetcherTimerJitterCoefficient = NewGlobalFloatSetting(
2753
"history.ReplicationTaskFetcherTimerJitterCoefficient",
2754
0.15,
2755
`ReplicationTaskFetcherTimerJitterCoefficient is the jitter for fetcher timer`,
2756
)
2757
ReplicationTaskFetcherErrorRetryWait = NewGlobalDurationSetting(
2758
"history.ReplicationTaskFetcherErrorRetryWait",
2759
time.Second,
2760
`ReplicationTaskFetcherErrorRetryWait is the wait time when fetcher encounters error`,
2761
)
2762
ReplicationTaskProcessorErrorRetryWait = NewShardIDDurationSetting(
2763
"history.ReplicationTaskProcessorErrorRetryWait",
2764
1*time.Second,
2765
`ReplicationTaskProcessorErrorRetryWait is the initial retry wait when we see errors in applying replication tasks`,
2766
)
2767
ReplicationTaskProcessorErrorRetryBackoffCoefficient = NewShardIDFloatSetting(
2768
"history.ReplicationTaskProcessorErrorRetryBackoffCoefficient",
2769
1.2,
2770
`ReplicationTaskProcessorErrorRetryBackoffCoefficient is the retry wait backoff time coefficient`,
2771
)
2772
ReplicationTaskProcessorErrorRetryMaxInterval = NewShardIDDurationSetting(
2773
"history.ReplicationTaskProcessorErrorRetryMaxInterval",
2774
5*time.Second,
2775
`ReplicationTaskProcessorErrorRetryMaxInterval is the retry wait backoff max duration`,
2776
)
2777
ReplicationTaskProcessorErrorRetryMaxAttempts = NewShardIDIntSetting(
2778
"history.ReplicationTaskProcessorErrorRetryMaxAttempts",
2779
80,
2780
`ReplicationTaskProcessorErrorRetryMaxAttempts is the max retry attempts for applying replication tasks`,
2781
)
2782
ReplicationTaskProcessorErrorRetryExpiration = NewShardIDDurationSetting(
2783
"history.ReplicationTaskProcessorErrorRetryExpiration",
2784
5*time.Minute,
2785
`ReplicationTaskProcessorErrorRetryExpiration is the max retry duration for applying replication tasks`,
2786
)
2787
ReplicationTaskProcessorNoTaskInitialWait = NewShardIDDurationSetting(
2788
"history.ReplicationTaskProcessorNoTaskInitialWait",
2789
2*time.Second,
2790
`ReplicationTaskProcessorNoTaskInitialWait is the wait time when not ask is returned`,
2791
)
2792
ReplicationTaskProcessorCleanupInterval = NewShardIDDurationSetting(
2793
"history.ReplicationTaskProcessorCleanupInterval",
2794
1*time.Minute,
2795
`ReplicationTaskProcessorCleanupInterval determines how frequently the cleanup replication queue`,
2796
)
2797
ReplicationTaskProcessorCleanupJitterCoefficient = NewShardIDFloatSetting(
2798
"history.ReplicationTaskProcessorCleanupJitterCoefficient",
2799
0.15,
2800
`ReplicationTaskProcessorCleanupJitterCoefficient is the jitter for cleanup timer`,
2801
)
2802
ReplicationTaskProcessorHostQPS = NewGlobalFloatSetting(
2803
"history.ReplicationTaskProcessorHostQPS",
2804
1500,
2805
`ReplicationTaskProcessorHostQPS is the qps of task processing rate limiter on host level`,
2806
)
2807
ReplicationTaskProcessorShardQPS = NewGlobalFloatSetting(
2808
"history.ReplicationTaskProcessorShardQPS",
2809
30,
2810
`ReplicationTaskProcessorShardQPS is the qps of task processing rate limiter on shard level`,
2811
)
2812
ReplicationEnableDLQMetrics = NewGlobalBoolSetting(
2813
"history.ReplicationEnableDLQMetrics",
2814
true,
2815
`ReplicationEnableDLQMetrics is the flag to emit DLQ metrics`,
2816
)
2817
ReplicationEnableUpdateWithNewTaskMerge = NewGlobalBoolSetting(
2818
"history.ReplicationEnableUpdateWithNewTaskMerge",
2819
false,
2820
`ReplicationEnableUpdateWithNewTaskMerge is the flag controlling whether replication task merging logic
2821
should be enabled for non continuedAsNew workflow UpdateWithNew case.`,
2822
)
2823
ReplicationMultipleBatches = NewGlobalBoolSetting(
2824
"history.ReplicationMultipleBatches",
2825
false,
2826
`ReplicationMultipleBatches is the flag to enable replication of multiple history event batches`,
2827
)
2828
HistoryTaskDLQEnabled = NewGlobalBoolSetting(
2829
"history.TaskDLQEnabled",
2830
true,
2831
`HistoryTaskDLQEnabled enables the history task DLQ. This applies to internal tasks like transfer and timer tasks.
2832
Do not turn this on if you aren't using Cassandra as the history task DLQ is not implemented for other databases.`,
2833
)
2834
HistoryTaskDLQUnexpectedErrorAttempts = NewGlobalIntSetting(
2835
"history.TaskDLQUnexpectedErrorAttempts",
2836
70, // 70 attempts takes about an hour
2837
`HistoryTaskDLQUnexpectedErrorAttempts is the number of task execution attempts before sending the task to DLQ.`,
2838
)
2839
HistoryTaskDLQInternalErrors = NewGlobalBoolSetting(
2840
"history.TaskDLQInternalErrors",
2841
false,
2842
`HistoryTaskDLQInternalErrors causes history task processing to send tasks failing with serviceerror.Internal to
2843
the dlq (or will drop them if not enabled)`,
2844
)
2845
HistoryTaskDLQErrorPattern = NewGlobalStringSetting(
2846
"history.TaskDLQErrorPattern",
2847
"",
2848
`HistoryTaskDLQErrorPattern specifies a regular expression. If a task processing error matches with this regex,
2849
that task will be sent to DLQ.`,
2850
)
2851
2852
MaxLocalParentWorkflowVerificationDuration = NewGlobalDurationSetting(
2853
"history.maxLocalParentWorkflowVerificationDuration",
2854
5*time.Minute,
2855
`MaxLocalParentWorkflowVerificationDuration controls the maximum duration to verify on the local cluster before requesting to resend parent workflow.`,
2856
)
2857
2858
ReplicationStreamSyncStatusDuration = NewGlobalDurationSetting(
2859
"history.ReplicationStreamSyncStatusDuration",
2860
1*time.Second,
2861
`ReplicationStreamSyncStatusDuration sync replication status duration`,
2862
)
2863
ReplicationProcessorSchedulerQueueSize = NewGlobalIntSetting(
2864
"history.ReplicationProcessorSchedulerQueueSize",
2865
128,
2866
`ReplicationProcessorSchedulerQueueSize is the replication task executor queue size`,
2867
)
2868
ReplicationProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2869
"history.ReplicationProcessorSchedulerWorkerCount",
2870
512,
2871
`ReplicationProcessorSchedulerWorkerCount is the replication task executor worker count`,
2872
)
2873
ReplicationLowPriorityProcessorSchedulerWorkerCount = NewGlobalIntSetting(
2874
"history.ReplicationLowPriorityProcessorSchedulerWorkerCount",
2875
128,
2876
`ReplicationLowPriorityProcessorSchedulerWorkerCount is the low priority replication task executor worker count`,
2877
)
2878
ReplicationLowPriorityTaskParallelism = NewGlobalIntSetting(
2879
"history.ReplicationLowPriorityTaskParallelism",
2880
1,
2881
`ReplicationLowPriorityTaskParallelism is the number of executions' low priority replication tasks that can be processed in parallel`,
2882
)
2883
2884
EnableReplicationTaskBatching = NewGlobalBoolSetting(
2885
"history.EnableReplicationTaskBatching",
2886
false,
2887
`EnableReplicationTaskBatching is a feature flag for batching replicate history event task`,
2888
)
2889
EnableReplicationTaskTieredProcessing = NewGlobalBoolSetting(
2890
"history.EnableReplicationTaskTieredProcessing",
2891
false,
2892
`EnableReplicationTaskTieredProcessing is a feature flag for enabling tiered replication task processing stack`,
2893
)
2894
ReplicationStreamSenderHighPriorityQPS = NewGlobalIntSetting(
2895
"history.ReplicationStreamSenderHighPriorityQPS",
2896
100,
2897
`Maximum number of high priority replication tasks that can be sent per second per shard`,
2898
)
2899
ReplicationStreamSenderLowPriorityQPS = NewGlobalIntSetting(
2900
"history.ReplicationStreamSenderLowPriorityQPS",
2901
100,
2902
`Maximum number of low priority replication tasks that can be sent per second per shard`,
2903
)
2904
ReplicationStreamEventLoopRetryMaxAttempts = NewGlobalIntSetting(
2905
"history.ReplicationStreamEventLoopRetryMaxAttempts",
2906
100, // 0 means retry forever
2907
`Max attempts for retrying replication stream event loop`,
2908
)
2909
ReplicationReceiverMaxOutstandingTaskCount = NewGlobalIntSetting(
2910
"history.ReplicationReceiverMaxOutstandingTaskCount",
2911
500,
2912
`Maximum number of outstanding tasks allowed for a single shard in the stream receiver`,
2913
)
2914
ReplicationReceiverSlowSubmissionLatencyThreshold = NewGlobalDurationSetting(
2915
"history.ReplicationReceiverSubmissionLatencyThreshold",
2916
1*time.Second,
2917
`Scheduler latency threshold for recording slow scheduler submission`,
2918
)
2919
ReplicationReceiverSlowSubmissionWindow = NewGlobalDurationSetting(
2920
"history.ReplicationReceiverSlowSubmissionWindow",
2921
10*time.Second,
2922
`Time window within which a slow submission will pause replication flow control`,
2923
)
2924
EnableReplicationReceiverSlowSubmissionFlowControl = NewGlobalBoolSetting(
2925
"history.EnableReplicationReceiverSlowSubmissionFlowControl",
2926
false,
2927
`Enable slow submission flow control check in replication receiver`,
2928
)
2929
ReplicationResendMaxBatchCount = NewGlobalIntSetting(
2930
"history.ReplicationResendMaxBatchCount",
2931
10,
2932
`Maximum number of resend events batch for a single replication request`,
2933
)
2934
ReplicationProgressCacheMaxSize = NewGlobalIntSetting(
2935
"history.ReplicationProgressCacheMaxSize",
2936
128000,
2937
`ReplicationProgressCacheMaxSize is the maximum number of entries in the replication progress cache`,
2938
)
2939
ReplicationProgressCacheTTL = NewGlobalDurationSetting(
2940
"history.ReplicationProgressCacheTTL",
2941
time.Hour,
2942
`ReplicationProgressCacheTTL is TTL of replication progress cache`,
2943
)
2944
ReplicationStreamSendEmptyTaskDuration = NewGlobalDurationSetting(
2945
"history.ReplicationStreamSendEmptyTaskDuration",
2946
time.Minute,
2947
`ReplicationStreamSendEmptyTaskDuration is the interval to sync status when there is no replication task`,
2948
)
2949
ReplicationStreamReceiverLivenessMultiplier = NewGlobalIntSetting(
2950
"history.ReplicationReceiverLivenessMultiplier",
2951
3,
2952
"ReplicationStreamSendEmptyTask is the multiplier of liveness check interval on stream receiver",
2953
)
2954
ReplicationStreamSenderLivenessMultiplier = NewGlobalIntSetting(
2955
"history.ReplicationStreamSenderLivenessMultiplier",
2956
10,
2957
"ReplicationStreamSenderLivenessMultiplier is the multiplier of liveness check interval on stream sender",
2958
)
2959
EnableHistoryReplicationRateLimiter = NewNamespaceBoolSetting(
2960
"history.EnableHistoryReplicationRateLimiter",
2961
false,
2962
"EnableHistoryReplicationRateLimiter is the feature flag to enable rate limiter on history event replication",
2963
)
2964
ReplicationEnableRateLimit = NewGlobalBoolSetting(
2965
"history.ReplicationEnableRateLimit",
2966
true,
2967
`ReplicationEnableRateLimit is the feature flag to enable replication global rate limiter`,
2968
)
2969
ReplicationEnableRateLimitShadowMode = NewGlobalBoolSetting(
2970
"history.ReplicationEnableRateLimitShadowMode",
2971
false,
2972
`ReplicationEnableRateLimitShadowMode enables shadow mode for replication rate limiter (emit metrics only, no throttling)`,
2973
)
2974
ReplicationStreamSenderErrorRetryWait = NewGlobalDurationSetting(
2975
"history.ReplicationStreamSenderErrorRetryWait",
2976
1*time.Second,
2977
`ReplicationStreamSenderErrorRetryWait is the initial retry wait when we see errors in sending replication tasks`,
2978
)
2979
ReplicationStreamSenderErrorRetryBackoffCoefficient = NewGlobalFloatSetting(
2980
"history.ReplicationStreamSenderErrorRetryBackoffCoefficient",
2981
1.2,
2982
`ReplicationStreamSenderErrorRetryBackoffCoefficient is the retry wait backoff time coefficient`,
2983
)
2984
ReplicationStreamSenderErrorRetryMaxInterval = NewGlobalDurationSetting(
2985
"history.ReplicationStreamSenderErrorRetryMaxInterval",
2986
3*time.Second,
2987
`ReplicationStreamSenderErrorRetryMaxInterval is the retry wait backoff max duration`,
2988
)
2989
ReplicationStreamSenderErrorRetryMaxAttempts = NewGlobalIntSetting(
2990
"history.ReplicationStreamSenderErrorRetryMaxAttempts",
2991
80,
2992
`ReplicationStreamSenderErrorRetryMaxAttempts is the max retry attempts for sending replication tasks`,
2993
)
2994
ReplicationStreamSenderErrorRetryExpiration = NewGlobalDurationSetting(
2995
"history.ReplicationStreamSenderErrorRetryExpiration",
2996
3*time.Minute,
2997
`ReplicationStreamSenderErrorRetryExpiration is the max retry duration for sending replication tasks`,
2998
)
2999
ReplicationExecutableTaskErrorRetryWait = NewGlobalDurationSetting(
3000
"history.ReplicationExecutableTaskErrorRetryWait",
3001
1*time.Second,
3002
`ReplicationExecutableTaskErrorRetryWait is the initial retry wait when we see errors in executing replication tasks`,
3003
)
3004
ReplicationExecutableTaskErrorRetryBackoffCoefficient = NewGlobalFloatSetting(
3005
"history.ReplicationExecutableTaskErrorRetryBackoffCoefficient",
3006
1.2,
3007
`ReplicationExecutableTaskErrorRetryBackoffCoefficient is the retry wait backoff time coefficient`,
3008
)
3009
ReplicationExecutableTaskErrorRetryMaxInterval = NewGlobalDurationSetting(
3010
"history.ReplicationExecutableTaskErrorRetryMaxInterval",
3011
5*time.Second,
3012
`ReplicationExecutableTaskErrorRetryMaxInterval is the retry wait backoff max duration`,
3013
)
3014
ReplicationExecutableTaskErrorRetryMaxAttempts = NewGlobalIntSetting(
3015
"history.ReplicationExecutableTaskErrorRetryMaxAttempts",
3016
80,
3017
`ReplicationExecutableTaskErrorRetryMaxAttempts is the max retry attempts for executing replication tasks`,
3018
)
3019
ReplicationExecutableTaskErrorRetryExpiration = NewGlobalDurationSetting(
3020
"history.ReplicationExecutableTaskErrorRetryExpiration",
3021
10*time.Minute,
3022
`ReplicationExecutableTaskErrorRetryExpiration is the max retry duration for executing replication tasks`,
3023
)
3024
WorkflowIdReuseMinimalInterval = NewNamespaceDurationSetting(
3025
"history.workflowIdReuseMinimalInterval",
3026
1*time.Second,
3027
`WorkflowIdReuseMinimalInterval is used for timing how soon users can create new workflow with the same workflow ID.`,
3028
)
3029
EnableWorkflowIdReuseStartTimeValidation = NewNamespaceBoolSetting(
3030
"history.enableWorkflowIdReuseStartTimeValidation",
3031
false,
3032
`If true, validate the start time of the old workflow is older than WorkflowIdReuseMinimalInterval when reusing workflow ID.`,
3033
)
3034
BusinessIDReuseRate = NewNamespaceIntSetting(
3035
"history.businessIDReuseRate",
3036
0,
3037
`BusinessIDReuseRate limits the rate of new execution creation per
3038
(namespace, businessID, archetype) tuple on a single history host. 0 = disabled (default).`,
3039
)
3040
BusinessIDReuseBurstRatio = NewNamespaceFloatSetting(
3041
"history.businessIDReuseBurstRatio",
3042
1.0,
3043
`BusinessIDReuseBurstRatio is the burst-to-rate ratio for the per-(namespace, businessID, archetype)
3044
start rate limiter. Burst = max(1, int(rps * ratio)). Default 1.0 (no burst above rate).`,
3045
)
3046
BusinessIDReuseLimiterCacheSize = NewGlobalIntSetting(
3047
"history.businessIDReuseLimiterCacheSize",
3048
10000,
3049
`BusinessIDReuseLimiterCacheSize is the max number of per-(namespace, businessID, archetype) rate limiters
3050
cached on a single history shard. Requires service restart to take effect.`,
3051
)
3052
BusinessIDReuseLimiterCacheTTL = NewGlobalDurationSetting(
3053
"history.businessIDReuseLimiterCacheTTL",
3054
60*time.Second,
3055
`BusinessIDReuseLimiterCacheTTL is the TTL for per-(namespace, businessID, archetype) rate limiter cache entries.
3056
Requires service restart to take effect.`,
3057
)
3058
HealthPersistenceLatencyFailure = NewGlobalFloatSetting(
3059
"history.healthPersistenceLatencyFailure",
3060
500,
3061
"History service health check on persistence average latency (millisecond) threshold",
3062
)
3063
HealthPersistenceErrorRatio = NewGlobalFloatSetting(
3064
"history.healthPersistenceErrorRatio",
3065
0.90,
3066
"History service health check on persistence error ratio",
3067
)
3068
HealthRPCLatencyFailure = NewGlobalFloatSetting(
3069
"history.healthRPCLatencyFailure",
3070
500,
3071
"History service health check on RPC average latency (millisecond) threshold",
3072
)
3073
HealthRPCErrorRatio = NewGlobalFloatSetting(
3074
"history.healthRPCErrorRatio",
3075
0.90,
3076
"History service health check on RPC error ratio",
3077
)
3078
HealthHistoryInitializationTime = NewGlobalDurationSetting(
3079
"history.healthHistoryInitializationTime",
3080
60*time.Second,
3081
"gRPC health server NOT_SERVING will be suppressed from DeepHealthCheck for this long")
3082
SendRawHistoryBetweenInternalServices = NewGlobalBoolSetting(
3083
"history.sendRawHistoryBetweenInternalServices",
3084
false,
3085
`SendRawHistoryBetweenInternalServices is whether to send raw history events between internal temporal services`,
3086
)
3087
// SendRawHistoryBytesToMatchingService controls which field is used when sending raw history
3088
// from history service to matching service. IMPORTANT: Only enable this flag after all services
3089
// (history, matching, frontend) are upgraded to a version that supports this feature.
3090
// NOTE: This flag only has effect when SendRawHistoryBetweenInternalServices is also enabled.
3091
// If SendRawHistoryBetweenInternalServices is false, this flag is ignored.
3092
SendRawHistoryBytesToMatchingService = NewGlobalBoolSetting(
3093
"history.sendRawHistoryBytesToMatchingService",
3094
false,
3095
`SendRawHistoryBytesToMatchingService controls whether to use the new raw_history_bytes field (21) instead of raw_history field (20) when sending history to matching service. Only enable after all services are upgraded. NOTE: This flag only has effect when SendRawHistoryBetweenInternalServices is also enabled.`,
3096
)
3097
3098
EnableChasm = NewNamespaceBoolSetting(
3099
"history.enableChasm",
3100
true,
3101
"Use real chasm tree implementation instead of the noop one",
3102
)
3103
3104
ChasmMaxInMemoryPureTasks = NewGlobalIntSetting(
3105
"history.chasmMaxInMemoryPureTasks",
3106
32,
3107
`ChasmMaxInMemoryPureTasks is the maximum number of physical pure tasks that can be held in memory for best effort task deletion.`,
3108
)
3109
3110
EnableCHASMSchedulerCreation = NewNamespaceBoolSetting(
3111
"history.enableCHASMSchedulerCreation",
3112
false,
3113
`EnableCHASMSchedulerCreation controls whether new schedules are created using the CHASM (V2) implementation
3114
instead of the existing (V1) implementation.`,
3115
)
3116
3117
CHASMSchedulerCreationRolloutPercent = NewNamespaceIntSetting(
3118
"history.chasmSchedulerCreationRolloutPercent",
3119
0,
3120
`CHASMSchedulerCreationRolloutPercent is the per-namespace percentage of new schedules that will be
3121
created on the CHASM (V2) implementation. This setting is only consulted when EnableCHASMSchedulerCreation is true and
3122
is re-evaluated on every CreateSchedule RPC.`,
3123
)
3124
3125
EnableCHASMSchedulerRouting = NewNamespaceBoolSetting(
3126
"history.enableCHASMSchedulerRouting",
3127
true,
3128
`EnableCHASMSchedulerRouting controls whether schedule RPCs are routed to the CHASM (V2) implementation
3129
first (with fallback to V1), excluding CreateSchedule.`,
3130
)
3131
3132
EnableCHASMSchedulerMigration = NewNamespaceBoolSetting(
3133
"history.enableCHASMSchedulerMigration",
3134
false,
3135
`EnableCHASMSchedulerMigration controls whether existing V1 schedules are automatically migrated
3136
to the CHASM (V2) implementation on active scheduler workflows.`,
3137
)
3138
3139
CHASMSchedulerMigrationRolloutPercent = NewNamespaceIntSetting(
3140
"history.chasmSchedulerMigrationRolloutPercent",
3141
0,
3142
`CHASMSchedulerMigrationRolloutPercent is the per-namespace percentage of V1 schedules that will be
3143
migrated to the CHASM (V2) implementation This setting is only consulted when
3144
EnableCHASMSchedulerMigration is true. The decision is re-evaluated when a
3145
scheduler workflow starts or continues-as-new.`,
3146
)
3147
3148
EnableCHASMSchedulerMigrationWithRunningWorkflows = NewNamespaceBoolSetting(
3149
"history.enableCHASMSchedulerMigrationWithRunningWorkflows",
3150
false,
3151
`EnableCHASMSchedulerMigrationWithRunningWorkflows, when set to false, prevents schedules with
3152
running workflows from being migrated. This works around a known bug in 3P SDKs involving updating
3153
existing workflows to attach callbacks.`,
3154
)
3155
3156
EnableCHASMSchedulerSentinels = NewNamespaceBoolSetting(
3157
"history.enableCHASMSchedulerSentinels",
3158
true,
3159
`EnableCHASMSchedulerSentinels enables ID-space collision sentinels, and must be enabled and propagated in advance of EnableCHASMSchedulerCreation.`,
3160
)
3161
3162
EnableCHASMCallbacks = NewNamespaceBoolSetting(
3163
"history.enableCHASMCallbacks",
3164
true,
3165
`Controls whether new callbacks are created using the CHASM implementation
3166
instead of the previous HSM backed implementation.`,
3167
)
3168
3169
EnableSignalWithStartFromWorkflow = NewNamespaceBoolSetting(
3170
"history.enableSignalWithStartFromWorkflow",
3171
false,
3172
`Controls whether signal with start from workflow is enabled.`,
3173
)
3174
3175
EnableCHASMSignalBacklinks = NewNamespaceBoolSetting(
3176
"history.enableCHASMSignalBacklinks",
3177
false,
3178
`Controls whether incoming signal request IDs are tracked in the CHASM IncomingSignals
3179
map to enable DescribeWorkflow to resolve RequestIDRef signal backlinks. Requires EnableChasm.
3180
Only enable once all servers in the fleet have been upgraded to a version that understands
3181
the IncomingSignals CHASM field.`,
3182
)
3183
EnableWorkflowUpdateCallbacks = NewNamespaceBoolSetting(
3184
"history.enableUpdateCallbacks",
3185
false,
3186
`Controls whether completion callbacks are created for workflow updates using
3187
the CHASM implementation. When disabled, new update callbacks will not be registered,
3188
but existing callbacks will still be processed and fired.`,
3189
)
3190
3191
VersionMembershipCacheTTL = NewGlobalDurationSetting(
3192
"history.versionMembershipCacheTTL",
3193
1*time.Second,
3194
`TTL for caching RPC results that check whether a version is present in a task queue.`,
3195
)
3196
3197
VersionMembershipCacheMaxSize = NewGlobalIntSetting(
3198
"history.versionMembershipCacheMaxSize",
3199
10000,
3200
`Maximum number of entries in the version membership cache.`,
3201
)
3202
3203
ReactivationSignalDedupCacheMaxSize = NewGlobalIntSetting(
3204
"worker.reactivationSignalDedupCacheMaxSize",
3205
10000,
3206
`Maximum number of entries in the per-pod reactivation-signal dedup cache on the
3207
worker deployment client. Each entry tracks the highest revision signaled for one
3208
target version workflow.`,
3209
)
3210
3211
EnableVersionReactivationSignals = NewGlobalBoolSetting(
3212
"history.enableVersionReactivationSignals",
3213
false,
3214
`EnableVersionReactivationSignals controls whether reactivation signals are sent to version workflows
3215
when workflows are pinned to a potentially DRAINED/INACTIVE version. Set to false to disable signals
3216
globally if load becomes problematic.`,
3217
)
3218
3219
RoutingInfoCacheTTL = NewGlobalDurationSetting(
3220
"history.routingInfoCacheTTL",
3221
1*time.Second,
3222
`TTL for caching task queue routing info (deployment versions and ramping state).`,
3223
)
3224
3225
RoutingInfoCacheMaxSize = NewGlobalIntSetting(
3226
"history.routingInfoCacheMaxSize",
3227
10000,
3228
`Maximum number of entries in the routing info cache.`,
3229
)
3230
3231
ExternalPayloadsEnabled = NewNamespaceBoolSetting(
3232
"history.externalPayloadsEnabled",
3233
true,
3234
`ExternalPayloadsEnabled controls whether external payload features are enabled for a namespace.`,
3235
)
3236
3237
// keys for worker
3238
3239
WorkerPersistenceMaxQPS = NewGlobalIntSetting(
3240
"worker.persistenceMaxQPS",
3241
500,
3242
`WorkerPersistenceMaxQPS is the max qps worker host can query DB`,
3243
)
3244
WorkerPersistenceGlobalMaxQPS = NewGlobalIntSetting(
3245
"worker.persistenceGlobalMaxQPS",
3246
0,
3247
`WorkerPersistenceGlobalMaxQPS is the max qps worker cluster can query DB`,
3248
)
3249
WorkerPersistenceNamespaceMaxQPS = NewNamespaceIntSetting(
3250
"worker.persistenceNamespaceMaxQPS",
3251
0,
3252
`WorkerPersistenceNamespaceMaxQPS is the max qps each namespace on worker host can query DB`,
3253
)
3254
WorkerPersistenceGlobalNamespaceMaxQPS = NewNamespaceIntSetting(
3255
"worker.persistenceGlobalNamespaceMaxQPS",
3256
0,
3257
`WorkerPersistenceNamespaceMaxQPS is the max qps each namespace in worker cluster can query DB`,
3258
)
3259
WorkerPersistenceDynamicRateLimitingParams = NewGlobalTypedSetting(
3260
"worker.persistenceDynamicRateLimitingParams",
3261
DefaultDynamicRateLimitingParams,
3262
`WorkerPersistenceDynamicRateLimitingParams is a struct that contains all adjustable dynamic rate limiting params.
3263
Fields: Enabled, RefreshInterval, LatencyThreshold, ErrorThreshold, RateBackoffStepSize, RateIncreaseStepSize, RateMultiMin, RateMultiMax.
3264
See DynamicRateLimitingParams comments for more details.`,
3265
)
3266
WorkerIndexerConcurrency = NewGlobalIntSetting(
3267
"worker.indexerConcurrency",
3268
100,
3269
`WorkerIndexerConcurrency is the max concurrent messages to be processed at any given time`,
3270
)
3271
WorkerESProcessorNumOfWorkers = NewGlobalIntSetting(
3272
"worker.ESProcessorNumOfWorkers",
3273
2,
3274
`WorkerESProcessorNumOfWorkers is num of workers for esProcessor`,
3275
)
3276
WorkerESProcessorBulkActions = NewGlobalIntSetting(
3277
"worker.ESProcessorBulkActions",
3278
500,
3279
`WorkerESProcessorBulkActions is max number of requests in bulk for esProcessor`,
3280
)
3281
WorkerESProcessorBulkSize = NewGlobalIntSetting(
3282
"worker.ESProcessorBulkSize",
3283
16*1024*1024,
3284
`WorkerESProcessorBulkSize is max total size of bulk in bytes for esProcessor`,
3285
)
3286
WorkerESProcessorFlushInterval = NewGlobalDurationSetting(
3287
"worker.ESProcessorFlushInterval",
3288
1*time.Second,
3289
`WorkerESProcessorFlushInterval is flush interval for esProcessor`,
3290
)
3291
WorkerESProcessorAckTimeout = NewGlobalDurationSetting(
3292
"worker.ESProcessorAckTimeout",
3293
30*time.Second,
3294
`WorkerESProcessorAckTimeout is the timeout that store will wait to get ack signal from ES processor.
3295
Should be at least WorkerESProcessorFlushInterval+<time to process request>.`,
3296
)
3297
WorkerThrottledLogRPS = NewGlobalIntSetting(
3298
"worker.throttledLogRPS",
3299
20,
3300
`WorkerThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger`,
3301
)
3302
WorkerScannerMaxConcurrentActivityExecutionSize = NewGlobalIntSetting(
3303
"worker.ScannerMaxConcurrentActivityExecutionSize",
3304
10,
3305
`WorkerScannerMaxConcurrentActivityExecutionSize indicates worker scanner max concurrent activity execution size`,
3306
)
3307
WorkerScannerMaxConcurrentWorkflowTaskExecutionSize = NewGlobalIntSetting(
3308
"worker.ScannerMaxConcurrentWorkflowTaskExecutionSize",
3309
10,
3310
`WorkerScannerMaxConcurrentWorkflowTaskExecutionSize indicates worker scanner max concurrent workflow execution size`,
3311
)
3312
WorkerScannerMaxConcurrentActivityTaskPollers = NewGlobalIntSetting(
3313
"worker.ScannerMaxConcurrentActivityTaskPollers",
3314
8,
3315
`WorkerScannerMaxConcurrentActivityTaskPollers indicates worker scanner max concurrent activity pollers`,
3316
)
3317
WorkerScannerMaxConcurrentWorkflowTaskPollers = NewGlobalIntSetting(
3318
"worker.ScannerMaxConcurrentWorkflowTaskPollers",
3319
8,
3320
`WorkerScannerMaxConcurrentWorkflowTaskPollers indicates worker scanner max concurrent workflow pollers`,
3321
)
3322
ScannerPersistenceMaxQPS = NewGlobalIntSetting(
3323
"worker.scannerPersistenceMaxQPS",
3324
100,
3325
`ScannerPersistenceMaxQPS is the maximum rate of persistence calls from worker.Scanner`,
3326
)
3327
ExecutionScannerPerHostQPS = NewGlobalIntSetting(
3328
"worker.executionScannerPerHostQPS",
3329
10,
3330
`ExecutionScannerPerHostQPS is the maximum rate of calls per host from executions.Scanner`,
3331
)
3332
ExecutionScannerPerShardQPS = NewGlobalIntSetting(
3333
"worker.executionScannerPerShardQPS",
3334
1,
3335
`ExecutionScannerPerShardQPS is the maximum rate of calls per shard from executions.Scanner`,
3336
)
3337
ExecutionDataDurationBuffer = NewGlobalDurationSetting(
3338
"worker.executionDataDurationBuffer",
3339
time.Hour*24*90,
3340
`ExecutionDataDurationBuffer is the data TTL duration buffer of execution data`,
3341
)
3342
ExecutionScannerWorkerCount = NewGlobalIntSetting(
3343
"worker.executionScannerWorkerCount",
3344
8,
3345
`ExecutionScannerWorkerCount is the execution scavenger worker count`,
3346
)
3347
ExecutionScannerHistoryEventIdValidator = NewGlobalBoolSetting(
3348
"worker.executionEnableHistoryEventIdValidator",
3349
true,
3350
`ExecutionScannerHistoryEventIdValidator is the flag to enable history event id validator`,
3351
)
3352
TaskQueueScannerEnabled = NewGlobalBoolSetting(
3353
"worker.taskQueueScannerEnabled",
3354
true,
3355
`TaskQueueScannerEnabled indicates if task queue scanner should be started as part of worker.Scanner`,
3356
)
3357
BuildIdScavengerEnabled = NewGlobalBoolSetting(
3358
"worker.buildIdScavengerEnabled",
3359
false,
3360
`BuildIdScavengerEnabled indicates if the build id scavenger should be started as part of worker.Scanner`,
3361
)
3362
HistoryScannerEnabled = NewGlobalBoolSetting(
3363
"worker.historyScannerEnabled",
3364
true,
3365
`HistoryScannerEnabled indicates if history scanner should be started as part of worker.Scanner`,
3366
)
3367
ExecutionsScannerEnabled = NewGlobalBoolSetting(
3368
"worker.executionsScannerEnabled",
3369
false,
3370
`ExecutionsScannerEnabled indicates if executions scanner should be started as part of worker.Scanner. This flag has no effect when SQL persistence is used,
3371
because executions scanner support for SQL is not yet implemented.`,
3372
)
3373
HistoryScannerDataMinAge = NewGlobalDurationSetting(
3374
"worker.historyScannerDataMinAge",
3375
60*24*time.Hour,
3376
`HistoryScannerDataMinAge indicates the history scanner cleanup minimum age.`,
3377
)
3378
HistoryScannerVerifyRetention = NewGlobalBoolSetting(
3379
"worker.historyScannerVerifyRetention",
3380
true,
3381
`HistoryScannerVerifyRetention indicates if the history scavenger should verify data retention.
3382
When enabled, the scavenger will delete completed workflow execution data that are older than the namespace retention period plus worker.executionDataDurationBuffer.`,
3383
)
3384
EnableBatcherNamespace = NewNamespaceBoolSetting(
3385
"worker.enableNamespaceBatcher",
3386
true,
3387
`EnableBatcher decides whether to start new (per-namespace) batcher in our worker`,
3388
)
3389
BatcherRPS = NewNamespaceIntSetting(
3390
"worker.batcherRPS",
3391
50,
3392
`BatcherRPS controls number the rps of one batch operation`,
3393
)
3394
BatcherConcurrency = NewNamespaceIntSetting(
3395
"worker.batcherConcurrency",
3396
5,
3397
`BatcherConcurrency controls the concurrency of one batch or admin batch operation`,
3398
)
3399
AdminBatcherHostRPS = NewGlobalIntSetting(
3400
"worker.adminBatcherHostRPS",
3401
100,
3402
`AdminBatcherHostRPS controls the rps of all admin batch operations per host`,
3403
)
3404
AdminBatcherGlobalRPS = NewGlobalIntSetting(
3405
"worker.adminBatcherGlobalRPS",
3406
0,
3407
`AdminBatcherGlobalRPS controls the rps of all admin batch operations across all worker hosts.
3408
The configured value will be divided by the number of worker hosts to get the per host rps limit.
3409
0 means no global limit and each host will use AdminBatcherHostRPS.`,
3410
)
3411
WorkerParentCloseMaxConcurrentActivityExecutionSize = NewGlobalIntSetting(
3412
"worker.ParentCloseMaxConcurrentActivityExecutionSize",
3413
1000,
3414
`WorkerParentCloseMaxConcurrentActivityExecutionSize indicates worker parent close worker max concurrent activity execution size`,
3415
)
3416
WorkerParentCloseMaxConcurrentWorkflowTaskExecutionSize = NewGlobalIntSetting(
3417
"worker.ParentCloseMaxConcurrentWorkflowTaskExecutionSize",
3418
1000,
3419
`WorkerParentCloseMaxConcurrentWorkflowTaskExecutionSize indicates worker parent close worker max concurrent workflow execution size`,
3420
)
3421
WorkerParentCloseMaxConcurrentActivityTaskPollers = NewGlobalIntSetting(
3422
"worker.ParentCloseMaxConcurrentActivityTaskPollers",
3423
4,
3424
`WorkerParentCloseMaxConcurrentActivityTaskPollers indicates worker parent close worker max concurrent activity pollers`,
3425
)
3426
WorkerParentCloseMaxConcurrentWorkflowTaskPollers = NewGlobalIntSetting(
3427
"worker.ParentCloseMaxConcurrentWorkflowTaskPollers",
3428
4,
3429
`WorkerParentCloseMaxConcurrentWorkflowTaskPollers indicates worker parent close worker max concurrent workflow pollers`,
3430
)
3431
WorkerPerNamespaceWorkerCount = NewNamespaceIntSetting(
3432
"worker.perNamespaceWorkerCount",
3433
1,
3434
`WorkerPerNamespaceWorkerCount controls number of per-ns (scheduler, batcher, etc.) workers to run per namespace`,
3435
)
3436
WorkerPerNamespaceWorkerOptions = NewNamespaceTypedSetting(
3437
"worker.perNamespaceWorkerOptions",
3438
sdkworker.Options{},
3439
`WorkerPerNamespaceWorkerOptions are SDK worker options for per-namespace workers`,
3440
)
3441
WorkerPerNamespaceWorkerStartRate = NewGlobalFloatSetting(
3442
"worker.perNamespaceWorkerStartRate",
3443
10.0,
3444
`WorkerPerNamespaceWorkerStartRate controls how fast per-namespace workers can be started (workers/second)`,
3445
)
3446
WorkerEnableScheduler = NewNamespaceBoolSetting(
3447
"worker.enableScheduler",
3448
true,
3449
`WorkerEnableScheduler controls whether to start the worker for scheduled workflows`,
3450
)
3451
WorkerStickyCacheSize = NewGlobalIntSetting(
3452
"worker.stickyCacheSize",
3453
0,
3454
`WorkerStickyCacheSize controls the sticky cache size for SDK workers on worker nodes
3455
(shared between all workers in the process, cannot be changed after startup)`,
3456
)
3457
SchedulerNamespaceStartWorkflowRPS = NewNamespaceFloatSetting(
3458
"worker.schedulerNamespaceStartWorkflowRPS",
3459
30.0,
3460
`SchedulerNamespaceStartWorkflowRPS is the per-namespace limit for starting workflows by schedules`,
3461
)
3462
SchedulerLocalActivitySleepLimit = NewNamespaceDurationSetting(
3463
"worker.schedulerLocalActivitySleepLimit",
3464
5*time.Second,
3465
`How long to sleep within a local activity before pushing to workflow level sleep (don't make this
3466
close to or more than the workflow task timeout)`,
3467
)
3468
SchedulerSpecMaxIterations = NewGlobalIntSetting(
3469
"scheduler.specMaxIterations",
3470
2*7*24*60*60,
3471
`SchedulerSpecMaxIterations is the hard bound on how many excluded candidate times the
3472
scheduler evaluates while searching for a schedule's next action time before giving up with an
3473
error and stopping the schedule.`,
3474
)
3475
SchedulerSpecWarnIterations = NewGlobalIntSetting(
3476
"scheduler.specWarnIterations",
3477
24*60*60,
3478
`SchedulerSpecWarnIterations is how many excluded candidate times the scheduler evaluates
3479
while searching for a schedule's next action time before emitting a warning (metric + log). It
3480
is non-fatal: the search continues past this threshold.`,
3481
)
3482
WorkerDeleteNamespaceActivityLimits = NewGlobalTypedSetting(
3483
"worker.deleteNamespaceActivityLimitsConfig",
3484
sdkworker.Options{},
3485
`WorkerDeleteNamespaceActivityLimitsConfig is a struct with relevant sdkworker.Options
3486
settings for controlling remote activity concurrency for delete namespace workflows.
3487
Valid fields: MaxConcurrentActivityExecutionSize, TaskQueueActivitiesPerSecond,
3488
WorkerActivitiesPerSecond, MaxConcurrentActivityTaskPollers.
3489
`,
3490
)
3491
WorkerGenerateMigrationTaskViaFrontend = NewGlobalBoolSetting(
3492
"worker.generateMigrationTaskViaFrontend",
3493
false,
3494
`WorkerGenerateMigrationTaskViaFrontend controls whether to generate migration tasks via frontend admin service.`,
3495
)
3496
WorkerEnableHistoryRateLimiter = NewGlobalBoolSetting(
3497
"worker.enableHistoryRateLimiter",
3498
false,
3499
`WorkerEnableHistoryRateLimiter decides whether to generate migration tasks with history length rate limiter.`,
3500
)
3501
MaxUserMetadataSummarySize = NewNamespaceIntSetting(
3502
"limit.userMetadataSummarySize",
3503
400,
3504
`MaxUserMetadataSummarySize is the maximum size of user metadata summary payloads in bytes.`,
3505
)
3506
MaxUserMetadataDetailsSize = NewNamespaceIntSetting(
3507
"limit.userMetadataDetailsSize",
3508
20000,
3509
`MaxUserMetadataDetailsSize is the maximum size of user metadata details payloads in bytes.`,
3510
)
3511
3512
MaxServiceErrorMessageLength = NewGlobalIntSetting(
3513
"system.maxServiceErrorMessageLength",
3514
4000,
3515
"MaxServiceErrorMessageLength is the max length of service error message. If it's longer, it will be truncated.",
3516
)
3517
3518
LogAllReqErrors = NewNamespaceBoolSetting(
3519
"system.logAllReqErrors",
3520
false,
3521
`When set to true, logs all RPC/request errors for the namespace, not just unexpected ones.`,
3522
)
3523
3524
WorkflowRulesAPIsEnabled = NewNamespaceBoolSetting(
3525
"frontend.workflowRulesAPIsEnabled",
3526
false,
3527
`WorkflowRulesAPIsEnabled is a "feature enable" flag. `,
3528
)
3529
3530
MaxWorkflowRulesPerNamespace = NewNamespaceIntSetting(
3531
"frontend.maxWorkflowRulesPerNamespace",
3532
10,
3533
`Maximum number of workflow rules in a given namespace`,
3534
)
3535
3536
SlowRequestLoggingThreshold = NewGlobalDurationSetting(
3537
"rpc.slowRequestLoggingThreshold",
3538
5*time.Second,
3539
`SlowRequestLoggingThreshold is the threshold above which a gRPC request is considered slow and logged.`,
3540
)
3541
3542
WorkerHeartbeatsEnabled = NewNamespaceBoolSetting(
3543
"frontend.WorkerHeartbeatsEnabled",
3544
true,
3545
`WorkerHeartbeatsEnabled is a "feature enable" flag. It allows workers to send periodic heartbeats to the server.`,
3546
)
3547
3548
EnableCancelWorkerPollsOnShutdown = NewNamespaceBoolSetting(
3549
"frontend.enableCancelWorkerPollsOnShutdown",
3550
false,
3551
`EnableCancelWorkerPollsOnShutdown enables eager cancellation of outstanding polls when a worker shuts down.
3552
When enabled, ShutdownWorker will cancel all outstanding polls for the worker before processing,
3553
preventing task orphaning that can occur if tasks are dispatched to a shutting-down worker.`,
3554
)
3555
3556
EnableMatchingFanOutForPollCancellation = NewNamespaceBoolSetting(
3557
"frontend.enableMatchingFanOutForPollCancellation",
3558
false,
3559
`EnableMatchingFanOutForPollCancellation controls where poll cancellation fan-out happens.
3560
When enabled, frontend sends root partition only; matching fans out to all partitions.
3561
When disabled, frontend iterates partitions; matching handles each partition locally.
3562
Default is false for safe rollout: flip to true after both frontend and matching are deployed.`,
3563
)
3564
3565
// Deprecated: ListWorkersEnabled is no longer honored. ListWorkers and DescribeWorker APIs are
3566
// always enabled. The write path is gated by WorkerHeartbeatsEnabled.
3567
ListWorkersEnabled = NewNamespaceBoolSetting(
3568
"frontend.ListWorkersEnabled",
3569
true,
3570
`Deprecated: no longer honored. ListWorkers and DescribeWorker are always enabled.`,
3571
)
3572
3573
WorkerCommandsEnabled = NewNamespaceBoolSetting(
3574
"frontend.WorkerCommandsEnabled",
3575
false,
3576
`WorkerCommandsEnabled is a "feature enable" flag. It allows clients to send commands to the workers.`,
3577
)
3578
3579
PollerAutoscalingAutoEnroll = NewNamespaceBoolSetting(
3580
"frontend.pollerAutoscalingAutoEnroll",
3581
false,
3582
`When true, workers should use poller autoscaling by default unless explicitly configured otherwise.`,
3583
)
3584
3585
WorkflowPauseEnabled = NewNamespaceBoolSetting(
3586
"frontend.WorkflowPauseEnabled",
3587
false,
3588
`WorkflowPauseEnabled is a "feature enable" flag. When enabled it allows clients to pause workflows.`,
3589
)
3590
TimeSkippingEnabled = NewNamespaceBoolSetting(
3591
"frontend.TimeSkippingEnabled",
3592
false,
3593
`TimeSkippingEnabled is a "feature enable" flag. When enabled it allows clients to skip time in executions.`,
3594
)
3595
)