Skip to content

Commit

Permalink
field cardinality temp save
Browse files Browse the repository at this point in the history
  • Loading branch information
Thejas-bhat committed Dec 6, 2024
1 parent 0abb3e6 commit 2b54a8d
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 8 deletions.
14 changes: 10 additions & 4 deletions index/scorch/snapshot_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ type asynchSegmentResult struct {
dict segment.TermDictionary
dictItr segment.DictionaryIterator

index int
docs *roaring.Bitmap
cardinality int
index int
docs *roaring.Bitmap

postings segment.PostingsList

Expand Down Expand Up @@ -146,6 +147,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,

results := make(chan *asynchSegmentResult)
var totalBytesRead uint64
var fieldCardinality int64
for _, s := range is.segment {
go func(s *SegmentSnapshot) {
dict, err := s.segment.Dictionary(field)
Expand All @@ -155,6 +157,8 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
if dictStats, ok := dict.(segment.DiskStatsReporter); ok {
atomic.AddUint64(&totalBytesRead, dictStats.BytesRead())
}

atomic.AddInt64(&fieldCardinality, int64(dict.Cardinality()))
if randomLookup {
results <- &asynchSegmentResult{dict: dict}
} else {
Expand All @@ -166,9 +170,11 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,

var err error
rv := &IndexSnapshotFieldDict{
snapshot: is,
cursors: make([]*segmentDictCursor, 0, len(is.segment)),
snapshot: is,
cursors: make([]*segmentDictCursor, 0, len(is.segment)),
cardinality: int(fieldCardinality),
}

for count := 0; count < len(is.segment); count++ {
asr := <-results
if asr.err != nil && err == nil {
Expand Down
10 changes: 6 additions & 4 deletions index/scorch/snapshot_index_dict.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ type segmentDictCursor struct {
}

type IndexSnapshotFieldDict struct {
snapshot *IndexSnapshot
cursors []*segmentDictCursor
entry index.DictEntry
bytesRead uint64
cardinality int
bytesRead uint64

snapshot *IndexSnapshot
cursors []*segmentDictCursor
entry index.DictEntry
}

func (i *IndexSnapshotFieldDict) BytesRead() uint64 {
Expand Down

0 comments on commit 2b54a8d

Please sign in to comment.