Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MB-59102: Merging KNN Results #1910

Merged
merged 11 commits into from
Nov 21, 2023
Merged

MB-59102: Merging KNN Results #1910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
147574e
merge KNN
CascadingRadium Nov 16, 2023
8f0d8c1
fix scoring
CascadingRadium Nov 17, 2023
a58fd7f
fix divide-by-zero error in knn scoring
CascadingRadium Nov 17, 2023
ae5eaeb
merge KNN results from multiple segments
CascadingRadium Nov 17, 2023
33e194f
Merge branch 'unstable' into MB-59102
CascadingRadium Nov 20, 2023
18cb894
compressed dataset
CascadingRadium Nov 20, 2023
716e29b
pagination fix
CascadingRadium Nov 20, 2023
54685ca
revert partitioned flag
CascadingRadium Nov 20, 2023
0ef2e33
Refactor API name: modifyRequestSize -> adjustRequestSizeForKNN, etc.
abhinavdangeti Nov 20, 2023
0ee7015
add conjunction support
CascadingRadium Nov 21, 2023
671df18
use maxKNNScore
CascadingRadium Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion index_alias_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@ func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest
if len(i.indexes) == 1 {
return i.indexes[0].SearchInContext(ctx, req)
}

return MultiSearch(ctx, req, i.indexes...)
}

Expand Down Expand Up @@ -453,6 +452,10 @@ func MultiSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*Se
req.SearchAfter = req.SearchBefore
req.SearchBefore = nil
}
originalSize := req.Size
if len(indexes) > 1 {
req.Size = adjustRequestSizeForKNN(req, len(indexes))
}

// run search on each index in separate go routine
var waitGroup sync.WaitGroup
Expand Down Expand Up @@ -491,6 +494,7 @@ func MultiSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*Se
indexErrors[asr.Name] = asr.Err
}
}
req.Size = originalSize

// merge just concatenated all the hits
// now lets clean it up
Expand All @@ -504,6 +508,10 @@ func MultiSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*Se
}
}

if len(indexes) > 1 {
mergeKNNResults(req, sr)
}

sortFunc := req.SortFunc()
// sort all hits with the requested order
if len(req.Sort) > 0 {
Expand Down
6 changes: 4 additions & 2 deletions index_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
req.SearchAfter = nil
}

return &SearchResult{
rv := &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Expand All @@ -653,7 +653,9 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
MaxScore: coll.MaxScore(),
Took: searchDuration,
Facets: coll.FacetResults(),
}, nil
}
mergeKNNResults(req, rv)
return rv, nil
}

func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
Expand Down
Loading
Loading