field cardinality temp save

index/scorch/snapshot_index.go

@@ -42,8 +42,9 @@ type asynchSegmentResult struct {
 	dict    segment.TermDictionary
 	dictItr segment.DictionaryIterator
 
-	index int
-	docs  *roaring.Bitmap
+	cardinality int
+	index       int
+	docs        *roaring.Bitmap
 
 	postings segment.PostingsList
 
@@ -146,6 +147,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 
 	results := make(chan *asynchSegmentResult)
 	var totalBytesRead uint64
+	var fieldCardinality int64
 	for _, s := range is.segment {
 		go func(s *SegmentSnapshot) {
 			dict, err := s.segment.Dictionary(field)
@@ -155,6 +157,8 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 				if dictStats, ok := dict.(segment.DiskStatsReporter); ok {
 					atomic.AddUint64(&totalBytesRead, dictStats.BytesRead())
 				}
+
+				atomic.AddInt64(&fieldCardinality, int64(dict.Cardinality()))
 				if randomLookup {
 					results <- &asynchSegmentResult{dict: dict}
 				} else {
@@ -166,9 +170,11 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 
 	var err error
 	rv := &IndexSnapshotFieldDict{
-		snapshot: is,
-		cursors:  make([]*segmentDictCursor, 0, len(is.segment)),
+		snapshot:    is,
+		cursors:     make([]*segmentDictCursor, 0, len(is.segment)),
+		cardinality: int(fieldCardinality),
 	}
+
 	for count := 0; count < len(is.segment); count++ {
 		asr := <-results
 		if asr.err != nil && err == nil {

index/scorch/snapshot_index_dict.go

@@ -28,10 +28,12 @@ type segmentDictCursor struct {
 }
 
 type IndexSnapshotFieldDict struct {
-	snapshot  *IndexSnapshot
-	cursors   []*segmentDictCursor
-	entry     index.DictEntry
-	bytesRead uint64
+	cardinality int
+	bytesRead   uint64
+
+	snapshot *IndexSnapshot
+	cursors  []*segmentDictCursor
+	entry    index.DictEntry
 }
 
 func (i *IndexSnapshotFieldDict) BytesRead() uint64 {

index_impl.go

@@ -464,8 +464,8 @@ func (i *indexImpl) preSearch(ctx context.Context, req *SearchRequest, reader in
 			Total:      1,
 			Successful: 1,
 		},
-		Hits:          knnHits,
-		totalDocCount: count,
+		Hits:     knnHits,
+		docCount: count,
 	}, nil
 }
 

pre_search.go

@@ -47,16 +47,23 @@ func (k *knnPreSearchResultProcessor) finalize(sr *SearchResult) {
 
 // -----------------------------------------------------------------------------
 type bm25PreSearchResultProcessor struct {
-	docCount uint64 // bm25 specific stats
+	docCount         uint64 // bm25 specific stats
+	fieldCardinality map[string]uint64
 }
 
 func newBM25PreSearchResultProcessor() *bm25PreSearchResultProcessor {
-	return &bm25PreSearchResultProcessor{}
+	return &bm25PreSearchResultProcessor{
+		fieldCardinality: make(map[string]uint64),
+	}
 }
 
 // TODO How will this work for queries other than term queries?
 func (b *bm25PreSearchResultProcessor) add(sr *SearchResult, indexName string) {
-	b.docCount += (sr.totalDocCount)
+	b.docCount += (sr.docCount)
+
+	for field, cardinality := range sr.fieldCardinality {
+		b.fieldCardinality[field] += cardinality
+	}
 }
 
 func (b *bm25PreSearchResultProcessor) finalize(sr *SearchResult) {

search.go

@@ -444,9 +444,10 @@ type SearchResult struct {
 	MaxScore float64                        `json:"max_score"`
 	Took     time.Duration                  `json:"took"`
 	Facets   search.FacetResults            `json:"facets"`
+
 	// The following fields are applicable to BM25 preSearch
-	// todo add more fields beyond docCount
-	totalDocCount uint64
+	docCount         uint64
+	fieldCardinality map[string]uint64 // search_field -> cardinality
 }
 
 func (sr *SearchResult) Size() int {

search/searcher/search_term.go

@@ -57,18 +57,22 @@ func NewTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, te
 
 func newTermSearcherFromReader(ctx context.Context, indexReader index.IndexReader, reader index.TermFieldReader,
 	term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) {
-	count, ok := ctx.Value(search.BM25PreSearchDataKey).(uint64)
-	if !ok {
-		var err error
-		count, err = indexReader.DocCount()
-		if err != nil {
-			_ = reader.Close()
-			return nil, err
+	var count uint64
+	if ctx != nil {
+		ctxCount, ok := ctx.Value(search.BM25PreSearchDataKey).(uint64)
+		if !ok {
+			var err error
+			ctxCount, err = indexReader.DocCount()
+			if err != nil {
+				_ = reader.Close()
+				return nil, err
+			}
+		} else {
+			fmt.Printf("fetched from ctx \n")
 		}
-	} else {
-		fmt.Printf("fetched from ctx \n")
-	}
+		count = ctxCount
 
+	}
 	scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), options)
 	return &TermSearcher{
 		indexReader: indexReader,