go.temporal.io/server/common/metrics/config.go
566 LOC · 156 covered · 410 uncovered · 50 ranges · 3348 concepts · 16 introducers · 1294 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
package metrics
2
3
import (
4
"errors"
5
"fmt"
6
"maps"
7
"time"
8
9
"github.com/cactus/go-statsd-client/v5/statsd"
10
prom "github.com/prometheus/client_golang/prometheus"
11
"github.com/uber-go/tally/v4"
12
"github.com/uber-go/tally/v4/prometheus"
13
"go.temporal.io/server/common/log"
14
"go.temporal.io/server/common/log/tag"
15
statsdreporter "go.temporal.io/server/common/metrics/tally/statsd"
16
)
17
18
type (
19
// Config contains the config items for metrics subsystem
20
Config struct {
21
ClientConfig `yaml:"clientConfig,inline"`
22
23
// Statsd is the configuration for statsd reporter
24
Statsd *StatsdConfig `yaml:"statsd"`
25
// Prometheus is the configuration for prometheus reporter
26
Prometheus *PrometheusConfig `yaml:"prometheus"`
27
}
28
29
ClientConfig struct {
30
// Tags is the set of key-value pairs to be reported as part of every metric
31
Tags map[string]string `yaml:"tags"`
32
// ExcludeTags is a map from tag name string to tag values string list.
33
// Each value present in keys will have relevant tag value replaced with "_tag_excluded_"
34
// Each value in values list will white-list tag values to be reported as usual.
35
ExcludeTags map[string][]string `yaml:"excludeTags"`
36
// Prefix sets the prefix to all outgoing metrics
37
// When migrating from tally to opentelemetry and to be backward compatible with the existing metric names,
38
// if the prefix has a "_" suffix, add an additional "_" at the end.
39
// i.e. "temporal" -> "temporal", but "temporal_" -> "temporal__", "temporal__" -> "temporal___".
40
// This is because tally implementation blindly adds "_" as the separator between the prefix
41
// and the metric name, while opentelemetry implementation only adds it if it's not already there.
42
Prefix string `yaml:"prefix"`
43
44
// DefaultHistogramBoundaries defines the default histogram bucket
45
// boundaries.
46
// Configuration of histogram boundaries for given metric unit.
47
//
48
// Supported values:
49
// - "dimensionless"
50
// - "milliseconds"
51
// - "bytes"
52
PerUnitHistogramBoundaries map[string][]float64 `yaml:"perUnitHistogramBoundaries"`
53
54
// Following configs are added for backwards compatibility when switching from tally to opentelemetry
55
// All configs should be set to true when using opentelemetry framework to have the same behavior as tally.
56
57
// WithoutUnitSuffix controls the additional of unit suffixes to metric names.
58
// This config only takes effect when using opentelemetry framework.
59
// Note: this config only takes effect when using prometheus via opentelemetry framework
60
WithoutUnitSuffix bool `yaml:"withoutUnitSuffix"`
61
// WithoutCounterSuffix controls the additional of _total suffixes to counter metric names.
62
// This config only takes effect when using opentelemetry framework.
63
// Note: this config only takes effect when using prometheus via opentelemetry framework
64
WithoutCounterSuffix bool `yaml:"withoutCounterSuffix"`
65
// RecordTimerInSeconds controls if Timer metric should be emitted as number of seconds
66
// (instead of milliseconds).
67
// This config only takes effect when using prometheus via opentelemetry framework
68
RecordTimerInSeconds bool `yaml:"recordTimerInSeconds"`
69
// TagsCacheMaxSize controls the maximum number of entries in the metrics
70
// tag cache. When the cache is full, all entries are cleared. Default: 10000.
71
TagsCacheMaxSize int `yaml:"tagsCacheMaxSize"`
72
}
73
74
// StatsdConfig contains the config items for statsd metrics reporter
75
StatsdConfig struct {
76
// The host and port of the statsd server
77
HostPort string `yaml:"hostPort" validate:"nonzero"`
78
// The prefix to use in reporting to statsd
79
Prefix string `yaml:"prefix" validate:"nonzero"`
80
// FlushInterval is the maximum interval for sending packets.
81
// If it is not specified, it defaults to 1 second.
82
FlushInterval time.Duration `yaml:"flushInterval"`
83
// FlushBytes specifies the maximum udp packet size you wish to send.
84
// If FlushBytes is unspecified, it defaults to 1432 bytes, which is
85
// considered safe for local traffic.
86
FlushBytes int `yaml:"flushBytes"`
87
// Reporter allows additional configuration of the stats reporter, e.g. with custom tagging options.
88
Reporter StatsdReporterConfig `yaml:"reporter"`
89
// Metric framework: tally/opentelemetry. If not specified, it defaults to tally.
90
Framework string `yaml:"framework"`
91
}
92
93
StatsdReporterConfig struct {
94
// TagSeparator allows tags to be appended with a separator. If not specified tag keys and values
95
// are embedded to the stat name directly.
96
TagSeparator string `yaml:"tagSeparator"`
97
}
98
99
// PrometheusConfig is a new format for config for prometheus metrics.
100
PrometheusConfig struct {
101
// Metric framework: Tally/OpenTelemetry
102
Framework string `yaml:"framework"`
103
// Address for prometheus to serve metrics from.
104
ListenAddress string `yaml:"listenAddress"`
105
106
// HandlerPath if specified will be used instead of using the default
107
// HTTP handler path "/metrics".
108
HandlerPath string `yaml:"handlerPath"`
109
110
// LoggerRPS sets the RPS of the logger provided to prometheus. Default of 0 means no limit.
111
LoggerRPS float64 `yaml:"loggerRPS"`
112
113
// Configs below are kept for backwards compatibility with previously exposed tally prometheus.Configuration.
114
115
// Deprecated. ListenNetwork if specified will be used instead of using tcp network.
116
// Supported networks: tcp, tcp4, tcp6 and unix.
117
ListenNetwork string `yaml:"listenNetwork"`
118
119
// Deprecated. TimerType is the default Prometheus type to use for Tally timers.
120
// TimerType is always histogram.
121
TimerType string `yaml:"timerType"`
122
123
// Deprecated. Please use PerUnitHistogramBoundaries in ClientConfig.
124
// DefaultHistogramBoundaries defines the default histogram bucket boundaries for tally timer metrics.
125
DefaultHistogramBoundaries []float64 `yaml:"defaultHistogramBoundaries"`
126
127
// Deprecated. Please use PerUnitHistogramBoundaries in ClientConfig.
128
// DefaultHistogramBuckets if specified will set the default histogram
129
// buckets to be used by the reporter for tally timer metrics.
130
// The unit for value specified is Second.
131
// If specified, will override DefaultSummaryObjectives and PerUnitHistogramBoundaries["milliseconds"].
132
DefaultHistogramBuckets []HistogramObjective `yaml:"defaultHistogramBuckets"`
133
134
// Deprecated. DefaultSummaryObjectives if specified will set the default summary
135
// objectives to be used by the reporter.
136
// The unit for value specified is Second.
137
// If specified, will override PerUnitHistogramBoundaries["milliseconds"].
138
DefaultSummaryObjectives []SummaryObjective `yaml:"defaultSummaryObjectives"`
139
140
// Deprecated. OnError specifies what to do when an error either with listening
141
// on the specified listen address or registering a metric with the
142
// Prometheus. By default the registerer will panic.
143
OnError string `yaml:"onError"`
144
145
// Deprecated. SanitizeOptions is an optional field that enables a user to
146
// specify which characters are valid and/or should be replaced before metrics
147
// are emitted.
148
SanitizeOptions *SanitizeOptions `yaml:"sanitizeOptions"`
149
}
150
)
151
152
// Deprecated. HistogramObjective is a Prometheus histogram bucket.
153
// Added for backwards compatibility.
154
type HistogramObjective struct {
155
Upper float64 `yaml:"upper"`
156
}
157
158
// Deprecated. SummaryObjective is a Prometheus summary objective.
159
// Added for backwards compatibility.
160
type SummaryObjective struct {
161
Percentile float64 `yaml:"percentile"`
162
AllowedError float64 `yaml:"allowedError"`
163
}
164
165
type SanitizeRange struct {
166
StartRange string `yaml:"startRange"`
167
EndRange string `yaml:"endRange"`
168
}
169
170
type ValidCharacters struct {
171
Ranges []SanitizeRange `yaml:"ranges"`
172
SafeCharacters string `yaml:"safeChars"`
173
}
174
175
type SanitizeOptions struct {
176
NameCharacters *ValidCharacters `yaml:"nameChars"`
177
KeyCharacters *ValidCharacters `yaml:"keyChars"`
178
ValueCharacters *ValidCharacters `yaml:"valueChars"`
179
ReplacementCharacter string `yaml:"replacementChar"`
180
}
181
182
// Supported framework types
183
const (
184
// FrameworkTally tally framework id
185
FrameworkTally = "tally"
186
// FrameworkOpentelemetry OpenTelemetry framework id
187
FrameworkOpentelemetry = "opentelemetry"
188
)
189
190
// Valid unit name for PerUnitHistogramBoundaries config field
191
const (
192
UnitNameDimensionless = "dimensionless"
193
UnitNameMilliseconds = "milliseconds"
194
UnitNameBytes = "bytes"
195
)
196
197
// tally sanitizer options that satisfy both Prometheus and M3 restrictions.
198
// This will rename metrics at the tally emission level, so metrics name we
199
// use maybe different from what gets emitted. In the current implementation
200
// it will replace - and . with _
201
// We should still ensure that the base metrics are prometheus compatible,
202
// but this is necessary as the same prom client initialization is used by
203
// our system workflows.
204
var (
205
safeCharacters = []rune{'_'}
206
207
defaultTallySanitizeOptions = tally.SanitizeOptions{
208
NameCharacters: tally.ValidCharacters{
209
Ranges: tally.AlphanumericRange,
210
Characters: safeCharacters,
211
},
212
KeyCharacters: tally.ValidCharacters{
213
Ranges: tally.AlphanumericRange,
214
Characters: safeCharacters,
215
},
216
ValueCharacters: tally.ValidCharacters{
217
Ranges: tally.AlphanumericRange,
218
Characters: safeCharacters,
219
},
220
ReplacementCharacter: tally.DefaultReplacementCharacter,
221
}
222
223
defaultPerUnitHistogramBoundaries = map[string][]float64{
224
Dimensionless: {
225
1,
226
2,
227
5,
228
10,
229
20,
230
50,
231
100,
232
200,
233
500,
234
1_000,
235
2_000,
236
5_000,
237
10_000,
238
20_000,
239
50_000,
240
100_000,
241
},
242
Milliseconds: {
243
1,
244
2,
245
5,
246
10,
247
20,
248
50,
249
100,
250
200,
251
500,
252
1_000, // 1s
253
2_000,
254
5_000,
255
10_000, // 10s
256
20_000,
257
50_000,
258
100_000, // 100s = 1m40s
259
200_000,
260
500_000,
261
1_000_000, // 1000s = 16m40s
262
},
263
Bytes: {
264
1024,
265
2048,
266
4096,
267
8192,
268
16384,
269
32768,
270
65536,
271
131072,
272
262144,
273
524288,
274
1048576,
275
2097152,
276
4194304,
277
8388608,
278
16777216,
279
},
280
}
281
)
282
283
// NewScope builds a new tally scope for this metrics configuration
284
//
285
// If the underlying configuration is valid for multiple reporter types,
286
// only one of them will be used for reporting.
287
//
288
// Current priority order is:
289
// statsd > prometheus
291
>
if c.Statsd != nil {
293
>
}
296
>
if err != nil {
297
logger.Fatal("invalid sanitize options input on prometheus config", tag.Error(err))
298
return nil
299
}
300
302
logger = log.NewThrottledLogger(logger, func() float64 { return c.Prometheus.LoggerRPS })
303
}
304
306
>
logger,
307
>
convertPrometheusConfigToTally(&c.ClientConfig, c.Prometheus),
308
>
sanitizeOptions,
309
>
&c.ClientConfig,
310
>
)
311
}
313
}
314
315
>
func convertSanitizeOptionsToTally(config *PrometheusConfig) (tally.SanitizeOptions, error) {
config.go ×13
316
>
if config.SanitizeOptions == nil {
318
>
}
319
321
}
322
323
func convertPrometheusConfigToTally(
324
clientConfig *ClientConfig,
325
config *PrometheusConfig,
327
>
defaultObjectives := make([]prometheus.SummaryObjective, len(config.DefaultSummaryObjectives))
328
>
for i, item := range config.DefaultSummaryObjectives {
329
defaultObjectives[i].AllowedError = item.AllowedError
330
defaultObjectives[i].Percentile = item.Percentile
331
}
332
334
>
HandlerPath: config.HandlerPath,
335
>
ListenNetwork: config.ListenNetwork,
336
>
ListenAddress: config.ListenAddress,
337
>
TimerType: "histogram",
338
>
DefaultHistogramBuckets: buildTallyTimerHistogramBuckets(clientConfig, config),
339
>
DefaultSummaryObjectives: defaultObjectives,
340
>
OnError: config.OnError,
341
>
}
342
}
343
344
func buildTallyTimerHistogramBuckets(
345
clientConfig *ClientConfig,
346
config *PrometheusConfig,
348
>
if len(config.DefaultHistogramBuckets) > 0 {
349
result := make([]prometheus.HistogramObjective, len(config.DefaultHistogramBuckets))
350
for i, item := range config.DefaultHistogramBuckets {
351
result[i].Upper = item.Upper
352
}
353
return result
354
}
355
357
result := make([]prometheus.HistogramObjective, 0, len(config.DefaultHistogramBoundaries))
358
for _, value := range config.DefaultHistogramBoundaries {
359
result = append(result, prometheus.HistogramObjective{
360
Upper: value,
361
})
362
}
363
return result
364
}
365
367
>
result := make([]prometheus.HistogramObjective, 0, len(boundaries))
368
>
for _, boundary := range boundaries {
370
>
Upper: boundary / float64(time.Second/time.Millisecond), // convert milliseconds to seconds
371
>
})
372
>
}
374
}
375
377
>
buckets := maps.Clone(defaultPerUnitHistogramBoundaries)
378
>
379
>
// In config, when overwrite default buckets, we use [dimensionless / miliseconds / bytes] as keys.
380
>
// But in code, we use [1 / ms / By] as key (to align with otel unit definition). So we do conversion here.
381
>
if bucket, ok := clientConfig.PerUnitHistogramBoundaries[UnitNameDimensionless]; ok {
383
>
}
384
>
if bucket, ok := clientConfig.PerUnitHistogramBoundaries[UnitNameMilliseconds]; ok {
config.go ×4
386
>
}
388
buckets[Bytes] = bucket
389
}
390
392
>
for idx, boundary := range buckets[Milliseconds] {
393
>
bucketInSeconds[idx] = boundary / float64(time.Second/time.Millisecond)
394
>
}
395
>
buckets[Seconds] = bucketInSeconds
396
>
397
>
clientConfig.PerUnitHistogramBoundaries = buckets
398
}
399
400
// newStatsdScope returns a new statsd scope with
401
// a default reporting interval of a second
403
>
config := c.Statsd
404
>
if len(config.HostPort) == 0 {
405
return tally.NoopScope
406
}
408
>
Address: config.HostPort,
409
>
Prefix: config.Prefix,
410
>
FlushInterval: config.FlushInterval,
411
>
FlushBytes: config.FlushBytes,
412
>
})
413
>
if err != nil {
414
logger.Fatal("error creating statsd client", tag.Error(err))
415
}
416
// NOTE: according to (https://github.com/uber-go/tally) Tally's statsd implementation doesn't support tagging.
417
// Therefore, we implement Tally interface to have a statsd reporter that can support tagging
419
>
TagSeparator: c.Statsd.Reporter.TagSeparator,
420
>
}
421
>
reporter := statsdreporter.NewReporter(statter, opts)
422
>
scopeOpts := tally.ScopeOptions{
423
>
Tags: c.Tags,
424
>
Reporter: reporter,
425
>
Prefix: c.Prefix,
426
>
}
427
>
scope, _ := tally.NewRootScope(scopeOpts, time.Second)
428
>
return scope
429
}
430
431
// newPrometheusScope returns a new prometheus scope with
432
// a default reporting interval of a second
433
func newPrometheusScope(
434
logger log.Logger,
435
config *prometheus.Configuration,
436
sanitizeOptions tally.SanitizeOptions,
437
clientConfig *ClientConfig,
439
>
reporter, err := config.NewReporter(
440
>
prometheus.ConfigurationOptions{
441
>
Registry: prom.NewRegistry(),
442
>
OnError: func(err error) {
443
logger.Warn("error in prometheus reporter", tag.Error(err))
444
},
445
},
446
)
448
logger.Fatal("error creating prometheus reporter", tag.Error(err))
449
}
451
>
Tags: clientConfig.Tags,
452
>
CachedReporter: reporter,
453
>
Separator: prometheus.DefaultSeparator,
454
>
SanitizeOptions: &sanitizeOptions,
455
>
Prefix: clientConfig.Prefix,
456
>
}
457
>
scope, _ := tally.NewRootScope(scopeOpts, time.Second)
458
>
return scope
459
}
460
461
// MetricsHandlerFromConfig is used at startup to construct a MetricsHandler
463
>
if c == nil {
465
>
}
466
468
>
469
>
fatalOnListenerError := true
470
>
if c.Statsd != nil && c.Statsd.Framework == FrameworkOpentelemetry {
471
// create opentelemetry provider with just statsd
472
otelProvider, err := NewOpenTelemetryProviderWithStatsd(logger, c.Statsd, &c.ClientConfig)
473
if err != nil {
474
logger.Fatal(err.Error())
475
}
476
return NewOtelMetricsHandler(logger, otelProvider, c.ClientConfig, false)
477
}
478
481
>
otelProvider, err := NewOpenTelemetryProviderWithPrometheus(logger, c.Prometheus, &c.ClientConfig, fatalOnListenerError)
482
>
if err != nil {
483
logger.Fatal(err.Error())
484
}
485
>
return NewOtelMetricsHandler(logger, otelProvider, c.ClientConfig, c.RecordTimerInSeconds)
config.go ×2
486
}
487
488
// fallback to tally if no framework is specified
490
>
c.ClientConfig,
491
>
NewScope(logger, c),
492
>
), nil
493
}
494
496
>
tagsToFilter := make(map[string]map[string]struct{})
497
>
for key, val := range cfg.ExcludeTags {
499
>
for _, val := range val {
500
>
exclusions[val] = struct{}{}
501
>
}
502
>
tagsToFilter[key] = exclusions
503
}
505
}
506
508
>
startRangeRunes := []rune(s.StartRange)
509
>
if len(startRangeRunes) != 1 {
510
return tally.SanitizeRange{}, fmt.Errorf("start range '%+v' must be a single rune", startRangeRunes)
511
}
512
514
>
if len(endRangeRunes) != 1 {
515
return tally.SanitizeRange{}, fmt.Errorf("end range '%+v' must be a single rune", endRangeRunes)
516
}
517
519
}
520
522
>
var ranges []tally.SanitizeRange
523
>
524
>
for _, r := range v.Ranges {
525
>
tallyRange, err := r.toTally()
526
>
if err != nil {
527
return tally.ValidCharacters{}, err
528
}
529
531
}
532
534
>
Ranges: ranges,
535
>
Characters: []rune(v.SafeCharacters),
536
>
}, nil
537
}
538
540
>
tallyNameChars, err := s.NameCharacters.toTally()
541
>
if err != nil {
542
return tally.SanitizeOptions{}, fmt.Errorf("invalid nameChars: %v", err)
543
}
544
546
>
if err != nil {
547
return tally.SanitizeOptions{}, fmt.Errorf("invalid keyChars: %v", err)
548
}
549
551
>
if err != nil {
552
return tally.SanitizeOptions{}, fmt.Errorf("invalid valueChars: %v", err)
553
}
554
556
>
if len(replacementChars) != 1 {
557
return tally.SanitizeOptions{}, errors.New("can only specify a single replacement character")
558
}
559
561
>
NameCharacters: tallyNameChars,
562
>
KeyCharacters: tallyKeyChars,
563
>
ValueCharacters: tallyValueChars,
564
>
ReplacementCharacter: replacementChars[0],
565
>
}, nil
566
}