From f321aada8c0cea390a63586b53375a33c3312e17 Mon Sep 17 00:00:00 2001
From: Thejas-bhat <thejas.orkombu@couchbase.com>
Date: Fri, 3 Nov 2023 15:27:00 +0530
Subject: [PATCH] code cleanup

---
 go.mod                         | 12 +++++++-----
 go.sum                         | 16 ++++++++++------
 index/scorch/segment_plugin.go | 12 ++++++++----
 index_test.go                  | 33 +++++++++++++--------------------
 4 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/go.mod b/go.mod
index ff2e6ba9b..8a06f59ff 100644
--- a/go.mod
+++ b/go.mod
@@ -5,13 +5,13 @@ go 1.19
 require (
 	github.com/RoaringBitmap/roaring v1.2.3
 	github.com/bits-and-blooms/bitset v1.2.0
-	github.com/blevesearch/bleve_index_api v1.0.6
+	github.com/blevesearch/bleve_index_api v1.1.2
 	github.com/blevesearch/geo v0.1.18
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
 	github.com/blevesearch/gtreap v0.1.1
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.6
+	github.com/blevesearch/scorch_segment_api/v2 v2.2.2
 	github.com/blevesearch/segment v0.9.1
 	github.com/blevesearch/snowball v0.6.1
 	github.com/blevesearch/snowballstem v0.9.0
@@ -23,6 +23,7 @@ require (
 	github.com/blevesearch/zapx/v13 v13.3.10
 	github.com/blevesearch/zapx/v14 v14.3.10
 	github.com/blevesearch/zapx/v15 v15.3.13
+	github.com/blevesearch/zapx/v16 v16.0.0-00010101000000-000000000000
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.7.0
@@ -31,8 +32,8 @@ require (
 )
 
 require (
+	github.com/blevesearch/go-faiss v1.0.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/zapx v0.0.0-00010101000000-000000000000 // indirect
 	github.com/couchbase/ghistogram v0.1.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
@@ -40,7 +41,8 @@ require (
 	github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/sys v0.13.0 // indirect
 )
 
-replace github.com/blevesearch/zapx => ../zapx
+replace github.com/blevesearch/zapx/v16 => ../zapx
diff --git a/go.sum b/go.sum
index 90ebfb29b..8f1af17c7 100644
--- a/go.sum
+++ b/go.sum
@@ -2,10 +2,12 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO
 github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
 github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8=
-github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
+github.com/blevesearch/bleve_index_api v1.1.2 h1:A8MhXiNbZ9DI+lZytWkOY75MwesRdOlE+7/MzdC9YXY=
+github.com/blevesearch/bleve_index_api v1.1.2/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
 github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
 github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
+github.com/blevesearch/go-faiss v1.0.1 h1:B0/FGdmcdxHIM0DRPyy4aWk0ZjMTFbCsmIzra77GAxE=
+github.com/blevesearch/go-faiss v1.0.1/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -17,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
 github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.2 h1:wRkJ2tAP0HsS4M2pK/qDkNV0PvUo33umcsjzLIgvVA4=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.2/go.mod h1:7mKEerrxzvfImS2pMvpfV5MGFYVcQ9NBTkBd/ApU7cI=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -83,14 +85,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go
index 4ed9bbf01..aa601223d 100644
--- a/index/scorch/segment_plugin.go
+++ b/index/scorch/segment_plugin.go
@@ -23,12 +23,12 @@ import (
 	"github.com/blevesearch/bleve/v2/geo"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 
-	zapv16 "github.com/blevesearch/zapx"
 	zapv11 "github.com/blevesearch/zapx/v11"
 	zapv12 "github.com/blevesearch/zapx/v12"
 	zapv13 "github.com/blevesearch/zapx/v13"
 	zapv14 "github.com/blevesearch/zapx/v14"
 	zapv15 "github.com/blevesearch/zapx/v15"
+	zapv16 "github.com/blevesearch/zapx/v16"
 )
 
 // SegmentPlugin represents the essential functions required by a package to plug in
@@ -113,11 +113,15 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) {
 func chooseSegmentPlugin(forcedSegmentType string,
 	forcedSegmentVersion uint32) (SegmentPlugin, error) {
 
+	// TBD: zap v16 can support the older file formats as well, however
+	// it breaks the TestForceVersion unit test in scorch_test.go
+	// commenting the code below to avoid any breakages.
+	//
 	// v16 and above are able to handle upgrade scenarios, so no need to load
 	// the force load the older plugins
-	if defaultSegmentPlugin.Version() >= 16 {
-		return defaultSegmentPlugin, nil
-	}
+	// if defaultSegmentPlugin.Version() >= 16 {
+	// 	return defaultSegmentPlugin, nil
+	// }
 
 	if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {
 		if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {
diff --git a/index_test.go b/index_test.go
index aeeca072e..561118317 100644
--- a/index_test.go
+++ b/index_test.go
@@ -504,7 +504,7 @@ func TestBytesRead(t *testing.T) {
 	}
 }
 
-func TestIndexSectionsNew(t *testing.T) {
+func TestIndexSectionsUpgrade(t *testing.T) {
 	tmpIndexPath := createTmpIndexPath(t)
 	defer cleanupTmpIndexPath(t, tmpIndexPath)
 
@@ -524,9 +524,15 @@ func TestIndexSectionsNew(t *testing.T) {
 
 	typeFieldMapping := NewTextFieldMapping()
 	typeFieldMapping.Store = false
+	typeFieldMapping.DocValues = true
 	documentMapping.AddFieldMappingsAt("type", typeFieldMapping)
 
-	idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
+	config := map[string]interface{}{
+		"forceSegmentVersion": 11,
+		"forceSegmentType":    "zap",
+	}
+
+	idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -546,32 +552,19 @@ func TestIndexSectionsNew(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to index batch %v\n", err)
 	}
+	typeFacet := NewFacetRequest("type", 2)
 	query := NewQueryStringQuery("united")
-	searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
-	res, err := idx.Search(searchRequest)
-	if err != nil {
-		t.Error(err)
-	}
 
-	fmt.Printf("the results from the search %v", res)
-}
-
-func TestIndexSectionsOpen(t *testing.T) {
-	idx, err := Open("/Users/thejasbhat/fts/data/beer-sample._default.bix_5c5ffb88b031ecd1_acbbef99.pindex")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	typeFacet := NewFacetRequest("type", 2)
-	query := NewQueryStringQuery("wine")
-	searchRequest := NewSearchRequestOptions(query, int(10), 0, false)
+	searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
 	searchRequest.AddFacet("types", typeFacet)
 	res, err := idx.Search(searchRequest)
 	if err != nil {
 		t.Error(err)
 	}
 
-	fmt.Printf("the search result for query %v is :\n %v\n", query, res)
+	if res.Facets["types"].Total != 9 {
+		t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total)
+	}
 }
 
 func getBatchFromData(idx Index, fileName string) (*Batch, error) {