fair_task_reader.go ×33

Frontier kind: Code frontier

unlabeled · c_9ccfdd9169b3

229 tests · 3395 LOC · 149 files · introduces 0 tests · 243 LOC · 5 files

Introduces — evidence that enters the hierarchy at this concept

Code
52 ranges243 lines · 5 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
541 ranges3395 lines · 149 files · Browse complete extent
All tests (intent)
229 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

5 files ranked by introduced lines: 243 introduced LOC across 52 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

go.temporal.io/server/service/matching/fair_task_reader.go 152 introduced LOC · 33 ranges

Open complete file

85 subqueue subqueueIndex,
86 initialAckLevel fairLevel,
87 > ) *fairTaskReader { fair_task_reader.go
88 > return &fairTaskReader{
89 > backlogMgr: backlogMgr,
90 > subqueue: subqueue,
91 > logger: backlogMgr.logger,
92 > retrier: backoff.NewRetrier(
93 > backoff.NewExponentialRetryPolicy(50*time.Millisecond).
94 > WithMaximumInterval(10*time.Second).
95 > WithExpirationInterval(backoff.NoInterval),
96 > clock.NewRealTimeSource(),
97 > ),
98 > throttleRetrier: backoff.NewRetrier(
99 > backoff.NewExponentialRetryPolicy(2*time.Second).
100 > WithMaximumInterval(30*time.Second).
101 > WithExpirationInterval(backoff.NoInterval),
102 > clock.NewRealTimeSource(),
103 > ),
104 > backlogAge: newBacklogAgeTracker(),
105 > addRetries: semaphore.NewWeighted(concurrentAddRetries),
106 >
107 > // ack manager
108 > outstandingTasks: *newFairLevelTreeMap(),
109 > readLevel: initialAckLevel,
110 > ackLevel: initialAckLevel,
111 > evictedAcks: *btree.NewBTreeGOptions(fairLevel.less, btree.Options{NoLocks: true}),
112 >
113 > // gc state
114 > lastGCTime: time.Now(),
115 > }
116 > }
117
118 > func (tr *fairTaskReader) Start() { fair_task_reader.go
119 > tr.lock.Lock()
120 > defer tr.lock.Unlock()
121 > tr.maybeReadTasksLocked()
122 > }
123
124 func (tr *fairTaskReader) getOldestBacklogTime() time.Time {
197 }
198
199 > func (tr *fairTaskReader) maybeReadTasksLocked() { fair_task_reader.go
200 > // If readPending is true, readTasksImpl is running and will check shouldReadMoreLocked
201 > // before it exits, so we'll definitely do another read if shouldReadMoreLocked is true.
202 > // We also abort here if we're in the middle of a backoff or shutting down.
203 > if tr.readPending || !tr.shouldReadMoreLocked() ||
204 > tr.backoffTimer != nil || tr.backlogMgr.tqCtx.Err() != nil {
205 > return
206 > }
207 > tr.readPending = true
208 > go tr.readTasksImpl()
209 }
210
211 > func (tr *fairTaskReader) shouldReadMoreLocked() bool { fair_task_reader.go
212 > if tr.atEnd {
213 > // If we have the whole backlog in memory, we don't need to read anything.
214 > return false
215 > } else if tr.loadedTasks > tr.backlogMgr.config.GetTasksReloadAt() {
216 // Too many loaded already. We'll get called again when loadedTasks drops.
217 return false
218 }
219 > return true fair_task_reader.go
220 }
221
222 > func (tr *fairTaskReader) readTasksImpl() { fair_task_reader.go
223 > var lastErr error
224 > for {
225 > tr.lock.Lock()
226 > if lastErr != nil || !tr.shouldReadMoreLocked() {
227 > break // with lock still held
228 }
229 > readLevel, loadedTasks := tr.readLevel, tr.loadedTasks fair_task_reader.go
230 > tr.lock.Unlock()
231 >
232 > lastErr = tr.readTaskBatch(readLevel, loadedTasks)
233 }
234
235 // note tr.lock is still held here!
236 > tr.readPending = false fair_task_reader.go
237 >
238 > // process any tasks that were written while readPending was true
239 > var newTasks []*internalTask
240 > if len(tr.newlyWrittenTasks) != 0 {
241 newTasks = tr.mergeTasksLocked(tr.newlyWrittenTasks, mergeWrite)
242 clear(tr.newlyWrittenTasks)
250 // If a backoff timer fired while readPending was still true, its maybeReadTasksLocked call
251 // was a no-op. Re-check now that readPending is false to avoid getting stuck.
252 > tr.maybeReadTasksLocked() fair_task_reader.go
253 >
254 > // unlock before calling addTaskToMatcher
255 > tr.lock.Unlock()
256 >
257 > for _, task := range newTasks {
258 tr.addTaskToMatcher(task)
259 }
260 }
261
262 > func (tr *fairTaskReader) readTaskBatch(readLevel fairLevel, loadedTasks int) error { fair_task_reader.go
263 > batchSize := tr.backlogMgr.config.GetTasksBatchSize() - loadedTasks
264 > readFrom := readLevel.max(fairLevel{pass: 1, id: 0}).inc()
265 > res, err := tr.backlogMgr.db.GetFairTasks(tr.backlogMgr.tqCtx, tr.subqueue, readFrom, batchSize)
266 > if err != nil {
267 // TODO: Should we ever stop retrying on db errors?
268 if tr.backlogMgr.signalIfFatal(err) || common.IsContextCanceledErr(err) {
275 return err
276 }
277 > tr.retrier.Reset() fair_task_reader.go
278 > tr.throttleRetrier.Reset()
279 >
280 > // If we got less than we asked for, we know we hit the end.
281 > // If there was a concurrent write such that we incorrectly think we hit the end here,
282 > // it will be held and processed after we're done reading, and maybe reset atEnd then.
283 > mode := mergeReadMiddle
284 > if len(res.Tasks) < batchSize {
285 > mode = mergeReadToEnd
286 > }
287
288 // Note: even if (especially if) len(tasks) == 0, we should go through the mergeTasks logic
290 // mergeTasksLocked where they'll be added as pre-acked (nil) entries so they advance the
291 // ack level and get GC'd.
292 > tr.mergeTasks(res.Tasks, mode) fair_task_reader.go
293 >
294 > return nil
295 }
296
374 }
375
376 > func (tr *fairTaskReader) mergeTasks(tasks []*persistencespb.AllocatedTaskInfo, mode mergeMode) { fair_task_reader.go
377 > tr.lock.Lock()
378 >
379 > if mode == mergeWrite && tr.readPending {
380 // concurrent write + read: hold the just-written tasks and merge them after we process
381 // the read.
385 }
386
387 > newTasks := tr.mergeTasksLocked(tasks, mode) fair_task_reader.go
388 >
389 > // Detect stuck reader: no tasks in memory, not at end, no read goroutine running, no
390 > // retry pending. In this state, written tasks go only to DB (filtered above readLevel)
391 > // and nothing will trigger a read. The root cause is still under investigation.
392 > // TODO: remove this once the root cause is found and fixed.
393 > if mode == mergeWrite && !tr.atEnd && tr.loadedTasks == 0 && !tr.readPending && tr.backoffTimer == nil {
394 metrics.FairReaderStuckDetected.With(tr.backlogMgr.metricsHandler).Record(1)
395 tr.backlogMgr.throttledLogger.Warn("fair task reader stuck: atEnd=false, loadedTasks=0, no read pending")
400
401 // unlock before calling addTaskToMatcher
402 > tr.lock.Unlock() fair_task_reader.go
403 >
404 > for _, task := range newTasks {
405 tr.addTaskToMatcher(task)
406 }
408
409 // nolint:revive,cognitive-complexity // will be simplified in the future
410 > func (tr *fairTaskReader) mergeTasksLocked(tasks []*persistencespb.AllocatedTaskInfo, mode mergeMode) []*internalTask { fair_task_reader.go
411 > // Collect (1) currently loaded tasks in the matcher plus (2) the tasks we just read/wrote; sorted by level.
412 >
413 > // (1) Note these values are *internalTask.
414 > merged := tr.outstandingTasks.Select(func(k, v any) bool {
415 _, ok := v.(*internalTask)
416 return ok
417 })
418 // (2) Note these values are *AllocatedTaskInfo.
419 > for _, t := range tasks { fair_task_reader.go
420 level := fairLevelFromAllocatedTask(t)
421 if !tr.ackLevel.less(level) {
440 // Take as many of those as we want to keep in memory. The ones that are not already in the
441 // matcher, we have to add to the matcher.
442 > batchSize := tr.backlogMgr.config.GetTasksBatchSize() fair_task_reader.go
443 > it := merged.Iterator()
444 > var highestLevel fairLevel
445 > tasks = tasks[:0] // reuse incoming slice to avoid an allocation
446 > for b := 0; b < batchSize && it.Next(); b++ {
447 if t, ok := it.Value().(*persistencespb.AllocatedTaskInfo); ok {
448 // new task we need to add to the matcher
452 }
453
454 > if highestLevel.id != 0 { fair_task_reader.go
455 // If we have any tasks at all in memory, set readLevel to the maximum of that set.
456 tr.readLevel = highestLevel
457 > } else { fair_task_reader.go
458 // Otherwise start reading at ack level next.
459 tr.readLevel = tr.ackLevel
462 // If there are remaining tasks in the merged set, they can't fit in memory. If they came
463 // from the tasks we just wrote, ignore them. If they came from matcher, remove them.
464 > evictedAnyTasks := false fair_task_reader.go
465 > for it.Next() {
466 evictedAnyTasks = true
467 if task, ok := it.Value().(*internalTask); ok {
483 // we may use these acks to increment our ack level across dropped ranges of tasks.
484 // Cache these evicted acks so we can skip them if we re-read them later.
485 > tr.outstandingTasks.Select(func(k, v any) bool { fair_task_reader.go
486 return v == nil && tr.readLevel.less(k.(fairLevel))
487 }).Each(func(k, v any) {
492 })
493 // Trim the cache to max size by removing highest levels.
494 > for tr.evictedAcks.Len() > evictedAcksCacheSize { fair_task_reader.go
495 tr.evictedAcks.PopMax()
496 }
497
498 > internalTasks := make([]*internalTask, 0, len(tasks)) fair_task_reader.go
499 > for _, t := range tasks {
500 level := fairLevelFromAllocatedTask(t)
501 if _, have := tr.evictedAcks.Delete(level); have {
529 // Advance the ack level past any pre-acked (nil) entries we just added: expired tasks and
530 // acks we re-inserted from the evicted-ack cache. Harmless if we added none.
531 > tr.advanceAckLevelLocked() fair_task_reader.go
532 >
533 > // Update atEnd:
534 > // If we did a read and didn't get to the end, we can't possibly be at the end.
535 > // Also if we evicted anything from memory, we can't either.
536 > // If we read to the end and didn't evict anything, then we know we're at the end.
537 > // Otherwise (i.e. on write) leave atEnd unchanged.
538 > if mode == mergeReadMiddle || evictedAnyTasks {
539 tr.atEnd = false
540 > } else if mode == mergeReadToEnd { fair_task_reader.go
541 > tr.atEnd = true
542 > }
543
544 // If we're at the end, then outstandingTasks is the whole queue so we can set count.
545 > if count := tr.knownCountLocked(); count >= 0 { fair_task_reader.go
546 > tr.backlogMgr.db.setKnownFairBacklogCount(tr.subqueue, count)
547 > }
548
549 > return internalTasks fair_task_reader.go
550
551 // TODO: fine-grained metrics for mergeTasks behavior:
589 }
590
591 > func (tr *fairTaskReader) ackLevelPinnedLocked() bool { fair_task_reader.go
592 > return tr.ackLevelPinnedByWriter || len(tr.newlyWrittenTasks) > 0
593 > }
594
595 // call this whenever new tasks are acked or when ackLevelPinnedLocked() may turn from true to
596 // false (i.e. when ackLevelPinnedByWriter is set to false or newlyWrittenTasks is cleared).
597 > func (tr *fairTaskReader) advanceAckLevelLocked() { fair_task_reader.go
598 > if tr.ackLevelPinnedLocked() {
599 return
600 }
601
602 // Adjust the ack level as far as we can
603 > var numAcked int64 fair_task_reader.go
604 > for {
605 > minLevel, v := tr.outstandingTasks.Min()
606 > if minLevel == nil {
607 break
608 } else if _, ok := v.(*internalTask); ok {
614 }
615
616 > if numAcked > 0 { fair_task_reader.go
617 tr.numToGC += int(numAcked)
618 tr.maybeGCLocked()
655 }
656
657 > func (tr *fairTaskReader) knownCountLocked() int64 { fair_task_reader.go
658 > if tr.atEnd {
659 > return int64(tr.loadedTasks)
660 > }
661 return -1
662 }
go.temporal.io/server/service/matching/fair_task_writer.go 34 introduced LOC · 7 ranges

Open complete file

56
57 // Start fairTaskWriter background goroutine.
58 > func (w *fairTaskWriter) Start() { fair_task_writer.go
59 > go w.taskWriterLoop()
60 > }
61
62 func (w *fairTaskWriter) appendTask(
141 }
142
143 > func (w *fairTaskWriter) initState() error { fair_task_writer.go
144 > state, err := w.renewLeaseWithRetry(foreverRetryPolicy, common.IsPersistenceTransientError)
145 > if err != nil {
146 w.backlogMgr.initState(taskQueueState{}, err)
147 return err
148 }
149 > w.taskIDBlock = rangeIDToTaskIDBlock(state.rangeID, w.config.RangeSize) fair_task_writer.go
150 > w.currentTaskIDBlock = w.taskIDBlock
151 > w.backlogMgr.initState(state, nil)
152 > return nil
153 }
154
155 > func (w *fairTaskWriter) taskWriterLoop() { fair_task_writer.go
156 > if w.initState() != nil {
157 return
158 }
160 // TODO: this will be out of phase with the timer in fairBacklogManagerImpl.periodicSync.
161 // can we align them better?
162 > persistFairnessKeys := time.NewTicker(w.config.UpdateAckInterval()).C fair_task_writer.go
163 >
164 > var reqs []*writeTaskRequest
165 > for {
166 > atomic.StoreInt64(&w.currentTaskIDBlock.start, w.taskIDBlock.start)
167 > atomic.StoreInt64(&w.currentTaskIDBlock.end, w.taskIDBlock.end)
168 >
169 > // prepare slice for reuse
170 > clear(reqs)
171 > reqs = reqs[:0]
172 >
173 > select {
174 case <-w.backlogMgr.tqCtx.Done():
175 return
230 retryPolicy backoff.RetryPolicy,
231 retryErrors backoff.IsRetryable,
232 > ) (taskQueueState, error) { fair_task_writer.go
233 > var newState taskQueueState
234 > op := func(ctx context.Context) (err error) {
235 > newState, err = w.db.RenewLease(ctx)
236 > return
237 > }
238 > metrics.LeaseRequestPerTaskQueueCounter.With(w.backlogMgr.metricsHandler).Record(1)
239 > err := backoff.ThrottleRetryContext(w.backlogMgr.tqCtx, op, retryPolicy, retryErrors)
240 > if err != nil {
241 metrics.LeaseFailurePerTaskQueueCounter.With(w.backlogMgr.metricsHandler).Record(1)
242 return newState, err
243 }
244 > return newState, nil fair_task_writer.go
245 }
246
go.temporal.io/server/service/matching/fair_backlog_manager.go 32 introduced LOC · 8 ranges

Open complete file

107 }
108
109 > func (c *fairBacklogManagerImpl) Start() { fair_backlog_manager.go
110 > c.taskWriter.Start()
111 > }
112
113 func (c *fairBacklogManagerImpl) Stop() {
133 }
134
135 > func (c *fairBacklogManagerImpl) initState(state taskQueueState, err error) { fair_backlog_manager.go
136 > defer c.initializedError.Set(struct{}{}, err)
137 >
138 > if err != nil {
139 // We can't recover from here without starting over, so unload the whole task queue.
140 // Skip final update since we never initialized.
146 // Pass scale info back to physical tq from unversioned (default) queue.
147 // This must be done before c.initializedError.Set().
148 > if c.queueKey().Partition().IsRoot() && !c.queueKey().IsVersioned() && !c.isDraining { fair_backlog_manager.go
149 c.pqMgr.StartScaleManager(state.scaleState)
150 }
151
152 > if state.otherHasTasks { fair_backlog_manager.go
153 c.pqMgr.SetupDraining()
154 }
155
156 > c.subqueueLock.Lock() fair_backlog_manager.go
157 > defer c.subqueueLock.Unlock()
158 >
159 > c.loadSubqueuesLocked(state.subqueues)
160 > go c.periodicSync()
161 }
162
166 }
167
168 > func (c *fairBacklogManagerImpl) loadSubqueuesLocked(subqueues []persistencespb.SubqueueInfo) { fair_backlog_manager.go
169 > // TODO(pri): This assumes that subqueues never shrinks, and priority/fairness index of
170 > // existing subqueues never changes. If we change that, this logic will need to change.
171 > for i := range subqueues {
172 > subqueueIdx := subqueueIndex(i)
173 > if i >= len(c.subqueues) {
174 > r := newFairTaskReader(c, subqueueIdx, fairLevelFromProto(subqueues[i].FairAckLevel))
175 > r.Start()
176 > c.subqueues = append(c.subqueues, r)
177 > }
178 > c.subqueuesByPriority[priorityKey(subqueues[i].Key.Priority)] = subqueueIdx
179 > c.priorityBySubqueue[subqueueIdx] = priorityKey(subqueues[i].Key.Priority)
180 }
181 }
216 }
217
218 > func (c *fairBacklogManagerImpl) periodicSync() { fair_backlog_manager.go
219 > for {
220 > select {
221 case <-c.tqCtx.Done():
222 return
395 }
396
397 > func (c *fairBacklogManagerImpl) queueKey() *PhysicalTaskQueueKey { fair_backlog_manager.go
398 > return c.pqMgr.QueueKey()
399 > }
400
401 func (c *fairBacklogManagerImpl) getDB() *taskQueueDB {
go.temporal.io/server/service/matching/db.go 18 introduced LOC · 2 ranges

Open complete file

390 // Use this to reset ApproximateBacklogCount when the backlog count is known, e.g. when you're
391 // read to the end of the backlog.
392 > func (db *taskQueueDB) setKnownFairBacklogCount(subqueue subqueueIndex, count int64) { db.go
393 > db.Lock()
394 > defer db.Unlock()
395 >
396 > if db.subqueues[subqueue].ApproximateBacklogCount != count {
397 db.lastChange = time.Now()
398 db.subqueues[subqueue].ApproximateBacklogCount = count
717 inclusiveMinLevel fairLevel,
718 batchSize int,
719 > ) (*persistence.GetTasksResponse, error) { db.go
720 > return db.store.GetTasks(ctx, &persistence.GetTasksRequest{
721 > NamespaceID: db.queue.NamespaceId(),
722 > TaskQueue: db.queue.PersistenceName(),
723 > TaskType: db.queue.TaskType(),
724 > InclusiveMinPass: inclusiveMinLevel.pass,
725 > InclusiveMinTaskID: inclusiveMinLevel.id,
726 > ExclusiveMaxTaskID: math.MaxInt64,
727 > Subqueue: int(subqueue),
728 > PageSize: batchSize,
729 > UseLimit: true,
730 > })
731 > }
732
733 // CompleteTasksLessThan deletes of tasks less than the given taskID. Limit is
go.temporal.io/server/service/matching/fair_level.go 7 introduced LOC · 2 ranges

Open complete file

47
48 // Returns the next highest fair level.
49 > func (a fairLevel) inc() fairLevel { fair_level.go
50 > return fairLevel{pass: a.pass, id: a.id + 1}
51 > }
52
53 func fairLevelFromAllocatedTask(t *persistencespb.AllocatedTaskInfo) fairLevel {
55 }
56
57 > func fairLevelFromProto(l *taskqueuespb.FairLevel) fairLevel { fair_level.go
58 > if l == nil {
59 > return fairLevel{}
60 > }
61 return fairLevel{pass: l.TaskPass, id: l.TaskId}
62 }