85
}
86
88
>
for _, host := range hosts {
89
>
go func(hostAddress string) {
90
>
resp, err := h.checkHost(ctx, hostAddress)
91
>
if err != nil {
93
>
State: enumsspb.HEALTH_STATE_NOT_SERVING,
94
>
Checks: []*healthspb.HealthCheck{
95
>
{
96
>
CheckType: health.CheckTypeHostAvailability,
97
>
State: enumsspb.HEALTH_STATE_NOT_SERVING,
98
>
Message: fmt.Sprintf("failed to reach host for health check: %v", err),
99
>
},
100
>
},
101
>
}
102
>
}
103
>
receiveCh <- hostResult{address: hostAddress, response: resp}
health_check.go
104
}(host.GetAddress())
105
}
106
108
>
var hostDeclinedServingCount float64
109
>
var hostDetails []*healthspb.HostHealthDetail
110
>
var exampleFailedHost *healthspb.HostHealthDetail
111
>
for range hosts {
112
>
result := <-receiveCh
113
>
state := result.response.GetState()
114
>
115
>
detail := &healthspb.HostHealthDetail{
116
>
Address: result.address,
117
>
State: state,
118
>
Checks: result.response.GetChecks(),
119
>
}
120
>
hostDetails = append(hostDetails, detail)
121
>
122
>
switch state {
123
case enumsspb.HEALTH_STATE_SERVING:
124
// Do nothing.
125
case enumsspb.HEALTH_STATE_DECLINED_SERVING:
126
hostDeclinedServingCount++
128
>
// NOT_SERVING, UNSPECIFIED, INTERNAL_ERROR, or any unknown state.
129
>
failedHostCount++
130
>
if exampleFailedHost == nil {
131
>
exampleFailedHost = detail
132
>
}
133
}
134
}
136
>
137
>
// Make sure that at lease 2 hosts must be not ready to trigger this check.
138
>
proportionOfDeclinedServiceHosts := ensureMinimumProportionOfHosts(h.hostDeclinedServingProportion(), len(hosts))
139
>
140
>
var overallState enumsspb.HealthState
141
>
hostDeclinedServingProportion := hostDeclinedServingCount / float64(len(hosts))
142
>
if hostDeclinedServingProportion > proportionOfDeclinedServiceHosts {
143
h.logger.Warn("health check exceeded host declined serving proportion threshold", tag.Float64("host declined serving proportion threshold", proportionOfDeclinedServiceHosts))
144
overallState = enumsspb.HEALTH_STATE_DECLINED_SERVING
146
>
failedHostCountProportion := failedHostCount / float64(len(hosts))
health_check.go
147
>
if failedHostCountProportion+hostDeclinedServingProportion > h.hostFailurePercentage() {
148
>
h.logger.Warn("health check exceeded host failure percentage threshold",
health_check.go
149
>
tag.Float64("host failure percentage threshold", h.hostFailurePercentage()),
150
>
tag.Float64("host failure percentage", failedHostCountProportion),
151
>
tag.Float64("host declined serving percentage", hostDeclinedServingProportion),
152
>
tag.NewStringTag("example_failed_host", failedHostSummary(exampleFailedHost)),
153
>
)
154
>
overallState = enumsspb.HEALTH_STATE_NOT_SERVING
156
overallState = enumsspb.HEALTH_STATE_SERVING
157
}
158
}
159
161
>
State: overallState,
162
>
ServiceDetail: &healthspb.ServiceHealthDetail{
163
>
Service: string(h.serviceName),
164
>
State: overallState,
165
>
Hosts: hostDetails,
166
>
},
167
>
}, nil
168
}
169
170
>
func (h *healthCheckerImpl) checkHost(ctx context.Context, hostAddress string) (resp *historyservice.DeepHealthCheckResponse, retErr error) {
health_check.go
171
>
defer log.CapturePanic(h.logger, &retErr)
172
>
173
>
resp, err := h.healthCheckFn(ctx, hostAddress)
174
>
if err != nil {
175
>
h.logger.Warn("failed to ping deep health check", tag.Error(err), tag.ServerName(string(h.serviceName)))
health_check.go
176
>
return nil, err
177
>
}
178
if resp == nil {
179
resp = &historyservice.DeepHealthCheckResponse{