337
}
338
339
>
func (m *StreamReceiverMonitorImpl) evaluateSingleStreamConnection(key *ClusterShardKeyPair, current *streamStatus, previous *streamStatus) bool {
stream_receiver_monitor.go
340
>
if previous == nil || current == nil { // cluster metadata change or shard movement could cause stream reconnect and status could be nil
341
return true
342
}
343
>
if previous.isTieredStackEnabled != current.isTieredStackEnabled { // there is tiered stack config change, wait until it becomes stable
stream_receiver_monitor.go
344
return true
345
}
346
>
checkIfMakeProgress := func(priority enumsspb.TaskPriority, currentAckLevel int64, currentMaxTaskId int64, previousAckLevel int64, previousMaxReplicationTaskId int64) bool {
stream_receiver_monitor.go
347
>
// 2 continuous data points where ACK level is not moving forward and ACK level is behind previous Max Replication taskId
348
>
if currentAckLevel == previousAckLevel && currentAckLevel < previousMaxReplicationTaskId {
349
>
m.Logger.Error(
350
>
fmt.Sprintf("%v replication is not making progress. previousAckLevel: %v, previousMaxTaskId: %v, currentAckLevel: %v, currentMaxTaskId: %v",
351
>
priority.String(), previousAckLevel, previousMaxReplicationTaskId, currentAckLevel, currentMaxTaskId),
352
>
tag.SourceShardID(key.Server.ShardID), tag.TargetCluster(strconv.Itoa(int(key.Client.ClusterID))), tag.TargetShardID(key.Client.ShardID))
353
>
metrics.ReplicationStreamStuck.With(m.MetricsHandler).Record(
354
>
int64(1),
355
>
metrics.FromClusterIDTag(key.Server.ClusterID),
356
>
metrics.ToClusterIDTag(key.Client.ClusterID),
357
>
)
358
>
return false
359
>
}
360
>
return true
361
}
362
364
>
return checkIfMakeProgress(enumsspb.TASK_PRIORITY_UNSPECIFIED, current.defaultAckLevel, current.maxReplicationTaskId, previous.defaultAckLevel, previous.maxReplicationTaskId)
365
>
}
366
>
highPriorityResult := checkIfMakeProgress(enumsspb.TASK_PRIORITY_HIGH, current.highPriorityAckLevel, current.maxReplicationTaskId, previous.highPriorityAckLevel, previous.maxReplicationTaskId)
367
>
lowPriorityResult := checkIfMakeProgress(enumsspb.TASK_PRIORITY_LOW, current.lowPriorityAckLevel, current.maxReplicationTaskId, previous.lowPriorityAckLevel, previous.maxReplicationTaskId)
368
>
return highPriorityResult && lowPriorityResult
369
}
370