go.temporal.io/server/service/frontend/service.go
568 LOC · 256 covered · 312 uncovered · 18 ranges · 437 concepts · 10 introducers · 205 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
package frontend
2
3
import (
4
"net"
5
"regexp"
6
"sync"
7
"time"
8
9
"go.temporal.io/api/operatorservice/v1"
10
"go.temporal.io/api/workflowservice/v1"
11
"go.temporal.io/server/api/adminservice/v1"
12
"go.temporal.io/server/chasm/lib/activity"
13
"go.temporal.io/server/chasm/lib/callback"
14
chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation"
15
"go.temporal.io/server/common/dynamicconfig"
16
"go.temporal.io/server/common/log"
17
"go.temporal.io/server/common/log/tag"
18
"go.temporal.io/server/common/membership"
19
"go.temporal.io/server/common/metrics"
20
"go.temporal.io/server/common/persistence/visibility/manager"
21
"go.temporal.io/server/common/retrypolicy"
22
"go.temporal.io/server/components/nexusoperations"
23
"google.golang.org/grpc"
24
"google.golang.org/grpc/health"
25
healthpb "google.golang.org/grpc/health/grpc_health_v1"
26
"google.golang.org/grpc/reflection"
27
)
28
29
// Config represents configuration for frontend service
30
type Config struct {
31
NumHistoryShards int32
32
PersistenceMaxQPS dynamicconfig.IntPropertyFn
33
PersistenceGlobalMaxQPS dynamicconfig.IntPropertyFn
34
PersistenceNamespaceMaxQPS dynamicconfig.IntPropertyFnWithNamespaceFilter
35
PersistenceGlobalNamespaceMaxQPS dynamicconfig.IntPropertyFnWithNamespaceFilter
36
PersistencePerShardNamespaceMaxQPS dynamicconfig.IntPropertyFnWithNamespaceFilter
37
PersistenceDynamicRateLimitingParams dynamicconfig.TypedPropertyFn[dynamicconfig.DynamicRateLimitingParams]
38
PersistenceQPSBurstRatio dynamicconfig.FloatPropertyFn
39
40
VisibilityPersistenceMaxReadQPS dynamicconfig.IntPropertyFn
41
VisibilityPersistenceMaxWriteQPS dynamicconfig.IntPropertyFn
42
VisibilityPersistenceSlowQueryThreshold dynamicconfig.DurationPropertyFn
43
VisibilityMaxPageSize dynamicconfig.IntPropertyFnWithNamespaceFilter
44
EnableReadFromSecondaryVisibility dynamicconfig.BoolPropertyFnWithNamespaceFilter
45
VisibilityEnableShadowReadMode dynamicconfig.BoolPropertyFn
46
VisibilityDisableOrderByClause dynamicconfig.BoolPropertyFnWithNamespaceFilter
47
VisibilityEnableManualPagination dynamicconfig.BoolPropertyFnWithNamespaceFilter
48
VisibilityEnableUnifiedQueryConverter dynamicconfig.BoolPropertyFn
49
VisibilityAllowList dynamicconfig.BoolPropertyFnWithNamespaceFilter
50
SuppressErrorSetSystemSearchAttribute dynamicconfig.BoolPropertyFnWithNamespaceFilter
51
52
HistoryMaxPageSize dynamicconfig.IntPropertyFnWithNamespaceFilter
53
RPS dynamicconfig.IntPropertyFn
54
GlobalRPS dynamicconfig.IntPropertyFn
55
OperatorRPSRatio dynamicconfig.FloatPropertyFn
56
57
NamespaceReplicationInducingAPIsRPS dynamicconfig.IntPropertyFn
58
MaxNamespaceRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
59
MaxNamespaceBurstRatioPerInstance dynamicconfig.FloatPropertyFnWithNamespaceFilter
60
MaxConcurrentLongRunningRequestsPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
61
MaxGlobalConcurrentLongRunningRequests dynamicconfig.IntPropertyFnWithNamespaceFilter
62
PollWaitForNamespaceRateLimitToken dynamicconfig.BoolPropertyFnWithNamespaceFilter
63
MaxNamespaceVisibilityRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
64
MaxNamespaceVisibilityBurstRatioPerInstance dynamicconfig.FloatPropertyFnWithNamespaceFilter
65
MaxNamespaceNamespaceReplicationInducingAPIsRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
66
MaxNamespaceNamespaceReplicationInducingAPIsBurstRatioPerInstance dynamicconfig.FloatPropertyFnWithNamespaceFilter
67
GlobalWorkerDeploymentReadRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
68
GlobalWorkerDeploymentReadBurstRatio dynamicconfig.FloatPropertyFnWithNamespaceFilter
69
GlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
70
InternalFEGlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
71
GlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
72
InternalFEGlobalNamespaceVisibilityRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
73
GlobalNamespaceNamespaceReplicationInducingAPIsRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
74
MaxIDLengthLimit dynamicconfig.IntPropertyFn
75
WorkerBuildIdSizeLimit dynamicconfig.IntPropertyFn
76
ReachabilityTaskQueueScanLimit dynamicconfig.IntPropertyFn
77
ReachabilityQueryBuildIdLimit dynamicconfig.IntPropertyFn
78
ReachabilityCacheOpenWFsTTL dynamicconfig.DurationPropertyFn
79
ReachabilityCacheClosedWFsTTL dynamicconfig.DurationPropertyFn
80
ReachabilityQuerySetDurationSinceDefault dynamicconfig.DurationPropertyFn
81
DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter
82
ShutdownDrainDuration dynamicconfig.DurationPropertyFn
83
ShutdownFailHealthCheckDuration dynamicconfig.DurationPropertyFn
84
85
MaxFairnessWeightOverrideConfigLimit dynamicconfig.IntPropertyFnWithTaskQueueFilter
86
87
MaxBadBinaries dynamicconfig.IntPropertyFnWithNamespaceFilter
88
89
// security protection settings
90
DisableListVisibilityByFilter dynamicconfig.BoolPropertyFnWithNamespaceFilter
91
92
// size limit system protection
93
BlobSizeLimitError dynamicconfig.IntPropertyFnWithNamespaceFilter
94
BlobSizeLimitWarn dynamicconfig.IntPropertyFnWithNamespaceFilter
95
MemoSizeLimitError dynamicconfig.IntPropertyFnWithNamespaceFilter
96
97
ThrottledLogRPS dynamicconfig.IntPropertyFn
98
99
// Namespace specific config
100
EnableNamespaceNotActiveAutoForwarding dynamicconfig.BoolPropertyFnWithNamespaceFilter
101
ForceNamespaceSelectedAPIAutoForwarding dynamicconfig.BoolPropertyFnWithNamespaceFilter
102
NamespaceMinRetentionLocal dynamicconfig.DurationPropertyFn
103
NamespaceMinRetentionGlobal dynamicconfig.DurationPropertyFn
104
105
SearchAttributesNumberOfKeysLimit dynamicconfig.IntPropertyFnWithNamespaceFilter
106
SearchAttributesSizeOfValueLimit dynamicconfig.IntPropertyFnWithNamespaceFilter
107
SearchAttributesTotalSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter
108
109
// DefaultWorkflowRetryPolicy represents default values for unset fields on a Workflow's
110
// specified RetryPolicy
111
DefaultWorkflowRetryPolicy dynamicconfig.TypedPropertyFnWithNamespaceFilter[retrypolicy.DefaultRetrySettings]
112
113
// VisibilityArchival system protection
114
VisibilityArchivalQueryMaxPageSize dynamicconfig.IntPropertyFn
115
116
// DEPRECATED
117
SendRawWorkflowHistory dynamicconfig.BoolPropertyFnWithNamespaceFilter
118
119
// DefaultWorkflowTaskTimeout the default workflow task timeout
120
DefaultWorkflowTaskTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter
121
122
// EnableServerVersionCheck disables periodic version checking performed by the frontend
123
EnableServerVersionCheck dynamicconfig.BoolPropertyFn
124
125
// EnableTokenNamespaceEnforcement enables enforcement that namespace in completion token matches namespace of the request
126
EnableTokenNamespaceEnforcement dynamicconfig.BoolPropertyFn
127
128
// ExposeAuthorizerErrors controls whether errors returned by the Authorizer will be wrapped with a PermissionDenied error.
129
ExposeAuthorizerErrors dynamicconfig.BoolPropertyFn
130
131
// gRPC keep alive options
132
// If a client pings too frequently, terminate the connection.
133
KeepAliveMinTime dynamicconfig.DurationPropertyFn
134
// Allow pings even when there are no active streams (RPCs)
135
KeepAlivePermitWithoutStream dynamicconfig.BoolPropertyFn
136
// Close the connection if a client is idle.
137
KeepAliveMaxConnectionIdle dynamicconfig.DurationPropertyFn
138
// Close the connection if it is too old.
139
KeepAliveMaxConnectionAge dynamicconfig.DurationPropertyFn
140
// Additive period after MaxConnectionAge after which the connection will be forcibly closed.
141
KeepAliveMaxConnectionAgeGrace dynamicconfig.DurationPropertyFn
142
// Ping the client if it is idle to ensure the connection is still active.
143
KeepAliveTime dynamicconfig.DurationPropertyFn
144
// Wait for the ping ack before assuming the connection is dead.
145
KeepAliveTimeout dynamicconfig.DurationPropertyFn
146
147
// RPS per every parallel delete executions activity.
148
// Total RPS is equal to DeleteNamespaceDeleteActivityRPS * DeleteNamespaceConcurrentDeleteExecutionsActivities.
149
// Default value is 100.
150
DeleteNamespaceDeleteActivityRPS dynamicconfig.IntPropertyFn
151
// Page size to read executions from visibility for delete executions activity.
152
// Default value is 1000.
153
DeleteNamespacePageSize dynamicconfig.IntPropertyFn
154
// Number of pages before returning ContinueAsNew from delete executions activity.
155
// Default value is 256.
156
DeleteNamespacePagesPerExecution dynamicconfig.IntPropertyFn
157
// Number of concurrent delete executions activities.
158
// Must be not greater than 256 and number of worker cores in the cluster.
159
// Default is 4.
160
DeleteNamespaceConcurrentDeleteExecutionsActivities dynamicconfig.IntPropertyFn
161
// Duration for how long namespace stays in database
162
// after all namespace resources (i.e. workflow executions) are deleted.
163
// Default is 0, means, namespace will be deleted immediately.
164
DeleteNamespaceNamespaceDeleteDelay dynamicconfig.DurationPropertyFn
165
166
// Enable schedule-related RPCs
167
EnableSchedules dynamicconfig.BoolPropertyFnWithNamespaceFilter
168
169
// Enable CHASM tree infrastructure
170
EnableChasm dynamicconfig.BoolPropertyFnWithNamespaceFilter
171
// Enable creation of new schedules on CHASM (V2) engine
172
EnableCHASMSchedulerCreation dynamicconfig.BoolPropertyFnWithNamespaceFilter
173
// Per-namespace percentage [0-100] of new schedules routed to CHASM when
174
// EnableCHASMSchedulerCreation is true. Default 0.
175
CHASMSchedulerCreationRolloutPercent dynamicconfig.IntPropertyFnWithNamespaceFilter
176
// Enable CHASM-first routing for schedule RPCs other than CreateSchedule
177
EnableCHASMSchedulerRouting dynamicconfig.BoolPropertyFnWithNamespaceFilter
178
// Enables ID-space collision sentinels, and must be enabled and propagated in
179
// advance of EnableCHASMSchedulerCreation.
180
EnableCHASMSchedulerSentinels dynamicconfig.BoolPropertyFnWithNamespaceFilter
181
182
// Enable deployment RPCs
183
EnableDeployments dynamicconfig.BoolPropertyFnWithNamespaceFilter
184
185
// Enable deployment version RPCs
186
EnableDeploymentVersions dynamicconfig.BoolPropertyFnWithNamespaceFilter
187
188
// Enable batcher RPCs
189
EnableBatcher dynamicconfig.BoolPropertyFnWithNamespaceFilter
190
// Batch operation dynamic configs
191
MaxConcurrentBatchOperation dynamicconfig.IntPropertyFnWithNamespaceFilter
192
MaxExecutionCountBatchOperation dynamicconfig.IntPropertyFnWithNamespaceFilter
193
// Admin Batch operation dynamic config
194
MaxConcurrentAdminBatchOperation dynamicconfig.IntPropertyFnWithNamespaceFilter
195
EnableBatchOperationsForStandaloneActivities dynamicconfig.BoolPropertyFnWithNamespaceFilter
196
197
EnableUpdateWorkflowExecution dynamicconfig.BoolPropertyFnWithNamespaceFilter
198
EnableUpdateWorkflowExecutionAsyncAccepted dynamicconfig.BoolPropertyFnWithNamespaceFilter
199
EnableWorkflowUpdateCallbacks dynamicconfig.BoolPropertyFnWithNamespaceFilter
200
NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute dynamicconfig.IntPropertyFnWithNamespaceFilter
201
202
EnableWorkerVersioningData dynamicconfig.BoolPropertyFnWithNamespaceFilter
203
EnableWorkerVersioningWorkflow dynamicconfig.BoolPropertyFnWithNamespaceFilter
204
EnableWorkerVersioningRules dynamicconfig.BoolPropertyFnWithNamespaceFilter
205
206
CallbackURLMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter
207
CallbackHeaderMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter
208
MaxCallbacksPerWorkflow dynamicconfig.IntPropertyFnWithNamespaceFilter
209
CallbackEndpointConfigs dynamicconfig.TypedPropertyFnWithNamespaceFilter[callback.AddressMatchRules]
210
211
MaxNexusOperationTokenLength dynamicconfig.IntPropertyFnWithNamespaceFilter
212
NexusRequestHeadersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp]
213
NexusForwardRequestUseEndpoint dynamicconfig.BoolPropertyFn
214
NexusOperationsMetricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig]
215
216
LinkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter
217
MaxLinksPerRequest dynamicconfig.IntPropertyFnWithNamespaceFilter
218
219
AdminEnableListHistoryTasks dynamicconfig.BoolPropertyFn
220
221
MaskInternalErrorDetails dynamicconfig.BoolPropertyFnWithNamespaceFilter
222
223
// Health check
224
HistoryHostErrorPercentage dynamicconfig.FloatPropertyFn
225
HistoryHostSelfErrorProportion dynamicconfig.FloatPropertyFn
226
227
LogAllReqErrors dynamicconfig.BoolPropertyFnWithNamespaceFilter
228
229
EnableEagerWorkflowStart dynamicconfig.BoolPropertyFnWithNamespaceFilter
230
231
WorkflowRulesAPIsEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
232
MaxWorkflowRulesPerNamespace dynamicconfig.IntPropertyFnWithNamespaceFilter
233
234
WorkerHeartbeatsEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
235
EnableCancelWorkerPollsOnShutdown dynamicconfig.BoolPropertyFnWithNamespaceFilter
236
EnableMatchingFanOutForPollCancellation dynamicconfig.BoolPropertyFnWithNamespaceFilter
237
NumTaskQueueReadPartitions dynamicconfig.IntPropertyFnWithTaskQueueFilter
238
WorkerCommandsEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
239
PollerAutoscalingAutoEnroll dynamicconfig.BoolPropertyFnWithNamespaceFilter
240
WorkflowPauseEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
241
TimeSkippingEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
242
StandaloneNexusOperationsEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter
243
EnableWorkflowTaskCompletionPagination dynamicconfig.BoolPropertyFnWithNamespaceFilter
244
245
HTTPAllowedHosts dynamicconfig.TypedPropertyFn[*regexp.Regexp]
246
AllowedExperiments dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]string]
247
248
// CHASM archetypes
249
Activity *activity.Config
250
}
251
252
// IsExperimentAllowed checks if an experiment is enabled for a given namespace in the dynamic config.
253
// Returns true if the experiment is explicitly listed or if "*" (wildcard)
254
// is present in the allowed experiments list.
255
>
func (c *Config) IsExperimentAllowed(experiment string, namespace string) bool {
service.go ×1
256
>
allowedExperiments := c.AllowedExperiments(namespace)
257
>
for _, allowed := range allowedExperiments {
260
>
}
261
}
263
}
264
265
// NewConfig returns new service config with default values
266
func NewConfig(
267
dc *dynamicconfig.Collection,
268
numHistoryShards int32,
270
>
return &Config{
271
>
NumHistoryShards: numHistoryShards,
272
>
PersistenceMaxQPS: dynamicconfig.FrontendPersistenceMaxQPS.Get(dc),
273
>
PersistenceGlobalMaxQPS: dynamicconfig.FrontendPersistenceGlobalMaxQPS.Get(dc),
274
>
PersistenceNamespaceMaxQPS: dynamicconfig.FrontendPersistenceNamespaceMaxQPS.Get(dc),
275
>
PersistenceGlobalNamespaceMaxQPS: dynamicconfig.FrontendPersistenceGlobalNamespaceMaxQPS.Get(dc),
276
>
PersistencePerShardNamespaceMaxQPS: dynamicconfig.DefaultPerShardNamespaceRPSMax,
277
>
PersistenceDynamicRateLimitingParams: dynamicconfig.FrontendPersistenceDynamicRateLimitingParams.Get(dc),
278
>
PersistenceQPSBurstRatio: dynamicconfig.PersistenceQPSBurstRatio.Get(dc),
279
>
280
>
VisibilityPersistenceMaxReadQPS: dynamicconfig.VisibilityPersistenceMaxReadQPS.Get(dc),
281
>
VisibilityPersistenceMaxWriteQPS: dynamicconfig.VisibilityPersistenceMaxWriteQPS.Get(dc),
282
>
VisibilityPersistenceSlowQueryThreshold: dynamicconfig.VisibilityPersistenceSlowQueryThreshold.Get(dc),
283
>
VisibilityMaxPageSize: dynamicconfig.FrontendVisibilityMaxPageSize.Get(dc),
284
>
EnableReadFromSecondaryVisibility: dynamicconfig.EnableReadFromSecondaryVisibility.Get(dc),
285
>
VisibilityEnableShadowReadMode: dynamicconfig.VisibilityEnableShadowReadMode.Get(dc),
286
>
VisibilityDisableOrderByClause: dynamicconfig.VisibilityDisableOrderByClause.Get(dc),
287
>
VisibilityEnableManualPagination: dynamicconfig.VisibilityEnableManualPagination.Get(dc),
288
>
VisibilityEnableUnifiedQueryConverter: dynamicconfig.VisibilityEnableUnifiedQueryConverter.Get(dc),
289
>
VisibilityAllowList: dynamicconfig.VisibilityAllowList.Get(dc),
290
>
SuppressErrorSetSystemSearchAttribute: dynamicconfig.SuppressErrorSetSystemSearchAttribute.Get(dc),
291
>
292
>
HistoryMaxPageSize: dynamicconfig.FrontendHistoryMaxPageSize.Get(dc),
293
>
RPS: dynamicconfig.FrontendRPS.Get(dc),
294
>
GlobalRPS: dynamicconfig.FrontendGlobalRPS.Get(dc),
295
>
OperatorRPSRatio: dynamicconfig.OperatorRPSRatio.Get(dc),
296
>
NamespaceReplicationInducingAPIsRPS: dynamicconfig.FrontendNamespaceReplicationInducingAPIsRPS.Get(dc),
297
>
298
>
MaxNamespaceRPSPerInstance: dynamicconfig.FrontendMaxNamespaceRPSPerInstance.Get(dc),
299
>
MaxNamespaceBurstRatioPerInstance: dynamicconfig.FrontendMaxNamespaceBurstRatioPerInstance.Get(dc),
300
>
MaxConcurrentLongRunningRequestsPerInstance: dynamicconfig.FrontendMaxConcurrentLongRunningRequestsPerInstance.Get(dc),
301
>
MaxGlobalConcurrentLongRunningRequests: dynamicconfig.FrontendGlobalMaxConcurrentLongRunningRequests.Get(dc),
302
>
PollWaitForNamespaceRateLimitToken: dynamicconfig.PollWaitForNamespaceRateLimitToken.Get(dc),
303
>
MaxNamespaceVisibilityRPSPerInstance: dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance.Get(dc),
304
>
MaxNamespaceVisibilityBurstRatioPerInstance: dynamicconfig.FrontendMaxNamespaceVisibilityBurstRatioPerInstance.Get(dc),
305
>
MaxNamespaceNamespaceReplicationInducingAPIsRPSPerInstance: dynamicconfig.FrontendMaxNamespaceNamespaceReplicationInducingAPIsRPSPerInstance.Get(dc),
306
>
MaxNamespaceNamespaceReplicationInducingAPIsBurstRatioPerInstance: dynamicconfig.FrontendMaxNamespaceNamespaceReplicationInducingAPIsBurstRatioPerInstance.Get(dc),
307
>
GlobalWorkerDeploymentReadRPS: dynamicconfig.FrontendGlobalWorkerDeploymentReadRPS.Get(dc),
308
>
GlobalWorkerDeploymentReadBurstRatio: dynamicconfig.FrontendGlobalWorkerDeploymentReadBurstRatio.Get(dc),
309
>
310
>
GlobalNamespaceRPS: dynamicconfig.FrontendGlobalNamespaceRPS.Get(dc),
311
>
InternalFEGlobalNamespaceRPS: dynamicconfig.InternalFrontendGlobalNamespaceRPS.Get(dc),
312
>
GlobalNamespaceVisibilityRPS: dynamicconfig.FrontendGlobalNamespaceVisibilityRPS.Get(dc),
313
>
InternalFEGlobalNamespaceVisibilityRPS: dynamicconfig.InternalFrontendGlobalNamespaceVisibilityRPS.Get(dc),
314
>
// Overshoot since these low rate limits don't work well in an uncoordinated global limiter.
315
>
GlobalNamespaceNamespaceReplicationInducingAPIsRPS: dynamicconfig.FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS.Get(dc),
316
>
317
>
MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc),
318
>
WorkerBuildIdSizeLimit: dynamicconfig.WorkerBuildIdSizeLimit.Get(dc),
319
>
ReachabilityTaskQueueScanLimit: dynamicconfig.ReachabilityTaskQueueScanLimit.Get(dc),
320
>
ReachabilityQueryBuildIdLimit: dynamicconfig.ReachabilityQueryBuildIdLimit.Get(dc),
321
>
ReachabilityCacheOpenWFsTTL: dynamicconfig.ReachabilityCacheOpenWFsTTL.Get(dc),
322
>
ReachabilityCacheClosedWFsTTL: dynamicconfig.ReachabilityCacheClosedWFsTTL.Get(dc),
323
>
ReachabilityQuerySetDurationSinceDefault: dynamicconfig.ReachabilityQuerySetDurationSinceDefault.Get(dc),
324
>
MaxBadBinaries: dynamicconfig.FrontendMaxBadBinaries.Get(dc),
325
>
DisableListVisibilityByFilter: dynamicconfig.DisableListVisibilityByFilter.Get(dc),
326
>
BlobSizeLimitError: dynamicconfig.BlobSizeLimitError.Get(dc),
327
>
BlobSizeLimitWarn: dynamicconfig.BlobSizeLimitWarn.Get(dc),
328
>
MemoSizeLimitError: dynamicconfig.MemoSizeLimitError.Get(dc),
329
>
ThrottledLogRPS: dynamicconfig.FrontendThrottledLogRPS.Get(dc),
330
>
ShutdownDrainDuration: dynamicconfig.FrontendShutdownDrainDuration.Get(dc),
331
>
ShutdownFailHealthCheckDuration: dynamicconfig.FrontendShutdownFailHealthCheckDuration.Get(dc),
332
>
EnableNamespaceNotActiveAutoForwarding: dynamicconfig.EnableNamespaceNotActiveAutoForwarding.Get(dc),
333
>
ForceNamespaceSelectedAPIAutoForwarding: dynamicconfig.ForceNamespaceSelectedAPIAutoForwarding.Get(dc),
334
>
NamespaceMinRetentionLocal: dynamicconfig.NamespaceMinRetentionLocal.Get(dc),
335
>
NamespaceMinRetentionGlobal: dynamicconfig.NamespaceMinRetentionGlobal.Get(dc),
336
>
SearchAttributesNumberOfKeysLimit: dynamicconfig.SearchAttributesNumberOfKeysLimit.Get(dc),
337
>
SearchAttributesSizeOfValueLimit: dynamicconfig.SearchAttributesSizeOfValueLimit.Get(dc),
338
>
SearchAttributesTotalSizeLimit: dynamicconfig.SearchAttributesTotalSizeLimit.Get(dc),
339
>
VisibilityArchivalQueryMaxPageSize: dynamicconfig.VisibilityArchivalQueryMaxPageSize.Get(dc),
340
>
DisallowQuery: dynamicconfig.DisallowQuery.Get(dc),
341
>
SendRawWorkflowHistory: dynamicconfig.SendRawWorkflowHistory.Get(dc),
342
>
DefaultWorkflowRetryPolicy: dynamicconfig.DefaultWorkflowRetryPolicy.Get(dc),
343
>
DefaultWorkflowTaskTimeout: dynamicconfig.DefaultWorkflowTaskTimeout.Get(dc),
344
>
EnableServerVersionCheck: dynamicconfig.EnableServerVersionCheck.Get(dc),
345
>
EnableTokenNamespaceEnforcement: dynamicconfig.EnableTokenNamespaceEnforcement.Get(dc),
346
>
ExposeAuthorizerErrors: dynamicconfig.ExposeAuthorizerErrors.Get(dc),
347
>
KeepAliveMinTime: dynamicconfig.KeepAliveMinTime.Get(dc),
348
>
KeepAlivePermitWithoutStream: dynamicconfig.KeepAlivePermitWithoutStream.Get(dc),
349
>
KeepAliveMaxConnectionIdle: dynamicconfig.KeepAliveMaxConnectionIdle.Get(dc),
350
>
KeepAliveMaxConnectionAge: dynamicconfig.KeepAliveMaxConnectionAge.Get(dc),
351
>
KeepAliveMaxConnectionAgeGrace: dynamicconfig.KeepAliveMaxConnectionAgeGrace.Get(dc),
352
>
KeepAliveTime: dynamicconfig.KeepAliveTime.Get(dc),
353
>
KeepAliveTimeout: dynamicconfig.KeepAliveTimeout.Get(dc),
354
>
355
>
DeleteNamespaceDeleteActivityRPS: dynamicconfig.DeleteNamespaceDeleteActivityRPS.Get(dc),
356
>
DeleteNamespacePageSize: dynamicconfig.DeleteNamespacePageSize.Get(dc),
357
>
DeleteNamespacePagesPerExecution: dynamicconfig.DeleteNamespacePagesPerExecution.Get(dc),
358
>
DeleteNamespaceConcurrentDeleteExecutionsActivities: dynamicconfig.DeleteNamespaceConcurrentDeleteExecutionsActivities.Get(dc),
359
>
DeleteNamespaceNamespaceDeleteDelay: dynamicconfig.DeleteNamespaceNamespaceDeleteDelay.Get(dc),
360
>
361
>
MaxFairnessWeightOverrideConfigLimit: dynamicconfig.MatchingMaxFairnessKeyWeightOverrides.Get(dc),
362
>
363
>
EnableSchedules: dynamicconfig.FrontendEnableSchedules.Get(dc),
364
>
EnableChasm: dynamicconfig.EnableChasm.Get(dc),
365
>
EnableCHASMSchedulerCreation: dynamicconfig.EnableCHASMSchedulerCreation.Get(dc),
366
>
CHASMSchedulerCreationRolloutPercent: dynamicconfig.CHASMSchedulerCreationRolloutPercent.Get(dc),
367
>
EnableCHASMSchedulerRouting: dynamicconfig.EnableCHASMSchedulerRouting.Get(dc),
368
>
EnableCHASMSchedulerSentinels: dynamicconfig.EnableCHASMSchedulerSentinels.Get(dc),
369
>
370
>
// [cleanup-wv-pre-release]
371
>
EnableDeployments: dynamicconfig.EnableDeployments.Get(dc),
372
>
EnableDeploymentVersions: dynamicconfig.EnableDeploymentVersions.Get(dc),
373
>
374
>
EnableBatcher: dynamicconfig.FrontendEnableBatcher.Get(dc),
375
>
MaxConcurrentBatchOperation: dynamicconfig.FrontendMaxConcurrentBatchOperationPerNamespace.Get(dc),
376
>
MaxExecutionCountBatchOperation: dynamicconfig.FrontendMaxExecutionCountBatchOperationPerNamespace.Get(dc),
377
>
MaxConcurrentAdminBatchOperation: dynamicconfig.FrontendMaxConcurrentAdminBatchOperationPerNamespace.Get(dc),
378
>
EnableBatchOperationsForStandaloneActivities: dynamicconfig.FrontendEnableBatchOperationsForStandaloneActivities.Get(dc),
379
>
380
>
EnableUpdateWorkflowExecution: dynamicconfig.FrontendEnableUpdateWorkflowExecution.Get(dc),
381
>
EnableUpdateWorkflowExecutionAsyncAccepted: dynamicconfig.FrontendEnableUpdateWorkflowExecutionAsyncAccepted.Get(dc),
382
>
EnableWorkflowUpdateCallbacks: dynamicconfig.EnableWorkflowUpdateCallbacks.Get(dc),
383
>
NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute: dynamicconfig.NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute.Get(dc),
384
>
385
>
EnableWorkerVersioningData: dynamicconfig.FrontendEnableWorkerVersioningDataAPIs.Get(dc),
386
>
EnableWorkerVersioningWorkflow: dynamicconfig.FrontendEnableWorkerVersioningWorkflowAPIs.Get(dc),
387
>
EnableWorkerVersioningRules: dynamicconfig.FrontendEnableWorkerVersioningRuleAPIs.Get(dc),
388
>
389
>
CallbackURLMaxLength: dynamicconfig.FrontendCallbackURLMaxLength.Get(dc),
390
>
CallbackHeaderMaxSize: dynamicconfig.FrontendCallbackHeaderMaxSize.Get(dc),
391
>
MaxCallbacksPerWorkflow: dynamicconfig.MaxCallbacksPerWorkflow.Get(dc),
392
>
MaxNexusOperationTokenLength: nexusoperations.MaxOperationTokenLength.Get(dc),
393
>
NexusRequestHeadersBlacklist: dynamicconfig.FrontendNexusRequestHeadersBlacklist.Get(dc),
394
>
NexusForwardRequestUseEndpoint: dynamicconfig.FrontendNexusForwardRequestUseEndpointDispatch.Get(dc),
395
>
NexusOperationsMetricTagConfig: nexusoperations.MetricTagConfiguration.Get(dc),
396
>
397
>
LinkMaxSize: dynamicconfig.FrontendLinkMaxSize.Get(dc),
398
>
MaxLinksPerRequest: dynamicconfig.FrontendMaxLinksPerRequest.Get(dc),
399
>
400
>
CallbackEndpointConfigs: callback.AllowedAddresses.Get(dc),
401
>
AdminEnableListHistoryTasks: dynamicconfig.AdminEnableListHistoryTasks.Get(dc),
402
>
403
>
MaskInternalErrorDetails: dynamicconfig.FrontendMaskInternalErrorDetails.Get(dc),
404
>
405
>
HistoryHostErrorPercentage: dynamicconfig.HistoryHostErrorPercentage.Get(dc),
406
>
HistoryHostSelfErrorProportion: dynamicconfig.HistoryHostSelfErrorProportion.Get(dc),
407
>
LogAllReqErrors: dynamicconfig.LogAllReqErrors.Get(dc),
408
>
EnableEagerWorkflowStart: dynamicconfig.EnableEagerWorkflowStart.Get(dc),
409
>
WorkflowRulesAPIsEnabled: dynamicconfig.WorkflowRulesAPIsEnabled.Get(dc),
410
>
MaxWorkflowRulesPerNamespace: dynamicconfig.MaxWorkflowRulesPerNamespace.Get(dc),
411
>
WorkerHeartbeatsEnabled: dynamicconfig.WorkerHeartbeatsEnabled.Get(dc),
412
>
EnableCancelWorkerPollsOnShutdown: dynamicconfig.EnableCancelWorkerPollsOnShutdown.Get(dc),
413
>
EnableMatchingFanOutForPollCancellation: dynamicconfig.EnableMatchingFanOutForPollCancellation.Get(dc),
414
>
NumTaskQueueReadPartitions: dynamicconfig.MatchingNumTaskqueueReadPartitions.Get(dc),
415
>
WorkerCommandsEnabled: dynamicconfig.WorkerCommandsEnabled.Get(dc),
416
>
PollerAutoscalingAutoEnroll: dynamicconfig.PollerAutoscalingAutoEnroll.Get(dc),
417
>
WorkflowPauseEnabled: dynamicconfig.WorkflowPauseEnabled.Get(dc),
418
>
TimeSkippingEnabled: dynamicconfig.TimeSkippingEnabled.Get(dc),
419
>
StandaloneNexusOperationsEnabled: chasmnexus.Enabled.Get(dc),
420
>
EnableWorkflowTaskCompletionPagination: dynamicconfig.EnableWorkflowTaskCompletionPagination.Get(dc),
421
>
422
>
HTTPAllowedHosts: dynamicconfig.FrontendHTTPAllowedHosts.Get(dc),
423
>
AllowedExperiments: dynamicconfig.FrontendAllowedExperiments.Get(dc),
424
>
425
>
Activity: activity.ConfigProvider(dc),
426
>
}
427
>
}
428
429
// Service represents the frontend service
430
type Service struct {
431
config *Config
432
433
healthServer *health.Server
434
handler Handler
435
adminHandler *AdminHandler
436
operatorHandler *OperatorHandlerImpl
437
versionChecker *VersionChecker
438
visibilityManager manager.VisibilityManager
439
server *grpc.Server
440
httpAPIServer *HTTPAPIServer
441
442
logger log.Logger
443
grpcListener net.Listener
444
metricsHandler metrics.Handler
445
membershipMonitor membership.Monitor
446
}
447
448
func NewService(
449
serviceConfig *Config,
450
server *grpc.Server,
451
healthServer *health.Server,
452
httpAPIServer *HTTPAPIServer,
453
handler Handler,
454
adminHandler *AdminHandler,
455
operatorHandler *OperatorHandlerImpl,
456
versionChecker *VersionChecker,
457
visibilityMgr manager.VisibilityManager,
458
logger log.Logger,
459
grpcListener net.Listener,
460
metricsHandler metrics.Handler,
461
membershipMonitor membership.Monitor,
463
>
return &Service{
464
>
config: serviceConfig,
465
>
server: server,
466
>
healthServer: healthServer,
467
>
httpAPIServer: httpAPIServer,
468
>
handler: handler,
469
>
adminHandler: adminHandler,
470
>
operatorHandler: operatorHandler,
471
>
versionChecker: versionChecker,
472
>
visibilityManager: visibilityMgr,
473
>
logger: logger,
474
>
grpcListener: grpcListener,
475
>
metricsHandler: metricsHandler,
476
>
membershipMonitor: membershipMonitor,
477
>
}
478
>
}
479
480
// Start starts the service
482
>
s.logger.Info("frontend starting")
483
>
484
>
healthpb.RegisterHealthServer(s.server, s.healthServer)
485
>
workflowservice.RegisterWorkflowServiceServer(s.server, s.handler)
486
>
adminservice.RegisterAdminServiceServer(s.server, s.adminHandler)
487
>
operatorservice.RegisterOperatorServiceServer(s.server, s.operatorHandler)
488
>
489
>
reflection.Register(s.server)
490
>
491
>
// must start resource first
492
>
metrics.RestartCount.With(s.metricsHandler).Record(1)
493
>
494
>
s.versionChecker.Start()
495
>
s.adminHandler.Start()
496
>
s.operatorHandler.Start()
497
>
s.handler.Start()
498
>
499
>
go func() {
500
>
s.logger.Info("Starting to serve on frontend listener")
501
>
if err := s.server.Serve(s.grpcListener); err != nil {
502
s.logger.Fatal("Failed to serve on frontend listener", tag.Error(err))
503
}
504
}()
505
508
>
if err := s.httpAPIServer.Serve(); err != nil {
509
s.logger.Fatal("Failed to serve HTTP API server", tag.Error(err))
510
}
511
}()
513
>
s.logger.Warn("HTTP API port has not been set. Nexus HTTP endpoints will not be available. " +
514
>
"To enable Nexus, follow these instructions: https://github.com/temporalio/temporal/blob/main/docs/architecture/nexus.md#enabling-nexus.")
515
>
}
516
518
}
519
520
// Stop stops the service
522
>
// initiate graceful shutdown:
523
>
// 1. Fail rpc health check, this will cause client side load balancer to stop forwarding requests to this node
524
>
// 2. wait for failure detection time
525
>
// 3. stop taking new requests by returning InternalServiceError
526
>
// 4. Wait for X second
527
>
// 5. Stop everything forcefully and return
528
>
529
>
requestDrainTime := max(time.Second, s.config.ShutdownDrainDuration())
530
>
failureDetectionTime := max(0, s.config.ShutdownFailHealthCheckDuration())
531
>
532
>
s.logger.Info("ShutdownHandler: Updating gRPC health status to ShuttingDown")
533
>
s.healthServer.Shutdown()
534
>
s.membershipMonitor.SetDraining(true)
535
>
536
>
s.logger.Info("ShutdownHandler: Waiting for others to discover I am unhealthy")
537
>
time.Sleep(failureDetectionTime)
538
>
539
>
s.handler.Stop()
540
>
s.operatorHandler.Stop()
541
>
s.adminHandler.Stop()
542
>
s.versionChecker.Stop()
543
>
s.visibilityManager.Close()
544
>
545
>
s.logger.Info("ShutdownHandler: Draining traffic")
546
>
// Gracefully stop gRPC server and HTTP API server concurrently
547
>
var wg sync.WaitGroup
548
>
wg.Go(func() {
549
>
t := time.AfterFunc(requestDrainTime, func() {
550
>
s.logger.Info("ShutdownHandler: Drain time expired, stopping all traffic")
service_grpc.pb.go ×19
551
>
s.server.Stop()
552
>
})
554
>
t.Stop()
555
})
558
>
s.httpAPIServer.GracefulStop(requestDrainTime)
559
>
})
560
}
562
>
563
>
if s.metricsHandler != nil {
564
>
s.metricsHandler.Stop(s.logger)
565
>
}
566
568
}