diff --git a/pkg/blockbuilder/scheduler/tracker.go b/pkg/blockbuilder/scheduler/tracker.go index e11ab088143e9..f320e500fe7e6 100644 --- a/pkg/blockbuilder/scheduler/tracker.go +++ b/pkg/blockbuilder/scheduler/tracker.go @@ -15,23 +15,6 @@ type Committer interface { Commit(ctx context.Context, partition int32, offset int64) error } -/* -why do we need a commit tracker? -given jobs generated by the planner can be processed parallelly and out of order, we need to keep track of the offset that have been completely consumed before committing them. -if there are gaps in what was consumed, tracker has to wait for those gaps to be consumed before committing. - -Start building with the assumption that the scheduler does not lose it's state - does not restart -We can write the completion messages to kafka in a follow-up to sustain restarts. - -read completion messages from the last 2h by default to recover the state and commit any missing offsets. -first thing to do before marking ready? - -what about the case when there are no commits? -planner and commit tracker should start from the same offset. -to keep things simple, we can assume that consumption start from start of the previous day if there are no commits. -This should ensure all components even across restarts are aligned. -*/ - type CommitTracker struct { mu sync.Mutex