From f321aada8c0cea390a63586b53375a33c3312e17 Mon Sep 17 00:00:00 2001 From: Thejas-bhat <thejas.orkombu@couchbase.com> Date: Fri, 3 Nov 2023 15:27:00 +0530 Subject: [PATCH] code cleanup --- go.mod | 12 +++++++----- go.sum | 16 ++++++++++------ index/scorch/segment_plugin.go | 12 ++++++++---- index_test.go | 33 +++++++++++++-------------------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index ff2e6ba9b..8a06f59ff 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.19 require ( github.com/RoaringBitmap/roaring v1.2.3 github.com/bits-and-blooms/bitset v1.2.0 - github.com/blevesearch/bleve_index_api v1.0.6 + github.com/blevesearch/bleve_index_api v1.1.2 github.com/blevesearch/geo v0.1.18 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/goleveldb v1.0.1 github.com/blevesearch/gtreap v0.1.1 - github.com/blevesearch/scorch_segment_api/v2 v2.1.6 + github.com/blevesearch/scorch_segment_api/v2 v2.2.2 github.com/blevesearch/segment v0.9.1 github.com/blevesearch/snowball v0.6.1 github.com/blevesearch/snowballstem v0.9.0 @@ -23,6 +23,7 @@ require ( github.com/blevesearch/zapx/v13 v13.3.10 github.com/blevesearch/zapx/v14 v14.3.10 github.com/blevesearch/zapx/v15 v15.3.13 + github.com/blevesearch/zapx/v16 v16.0.0-00010101000000-000000000000 github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 github.com/spf13/cobra v1.7.0 @@ -31,8 +32,8 @@ require ( ) require ( + github.com/blevesearch/go-faiss v1.0.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/zapx v0.0.0-00010101000000-000000000000 // indirect github.com/couchbase/ghistogram v0.1.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/snappy v0.0.1 // indirect @@ -40,7 +41,8 @@ require ( github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/sys v0.13.0 // indirect ) -replace github.com/blevesearch/zapx => ../zapx +replace github.com/blevesearch/zapx/v16 => ../zapx diff --git a/go.sum b/go.sum index 90ebfb29b..8f1af17c7 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,12 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= -github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/bleve_index_api v1.1.2 h1:A8MhXiNbZ9DI+lZytWkOY75MwesRdOlE+7/MzdC9YXY= +github.com/blevesearch/bleve_index_api v1.1.2/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/go-faiss v1.0.1 h1:B0/FGdmcdxHIM0DRPyy4aWk0ZjMTFbCsmIzra77GAxE= +github.com/blevesearch/go-faiss v1.0.1/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -17,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/scorch_segment_api/v2 v2.2.2 h1:wRkJ2tAP0HsS4M2pK/qDkNV0PvUo33umcsjzLIgvVA4= +github.com/blevesearch/scorch_segment_api/v2 v2.2.2/go.mod h1:7mKEerrxzvfImS2pMvpfV5MGFYVcQ9NBTkBd/ApU7cI= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A= @@ -83,14 +85,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index 4ed9bbf01..aa601223d 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -23,12 +23,12 @@ import ( "github.com/blevesearch/bleve/v2/geo" segment "github.com/blevesearch/scorch_segment_api/v2" - zapv16 "github.com/blevesearch/zapx" zapv11 "github.com/blevesearch/zapx/v11" zapv12 "github.com/blevesearch/zapx/v12" zapv13 "github.com/blevesearch/zapx/v13" zapv14 "github.com/blevesearch/zapx/v14" zapv15 "github.com/blevesearch/zapx/v15" + zapv16 "github.com/blevesearch/zapx/v16" ) // SegmentPlugin represents the essential functions required by a package to plug in @@ -113,11 +113,15 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) { func chooseSegmentPlugin(forcedSegmentType string, forcedSegmentVersion uint32) (SegmentPlugin, error) { + // TBD: zap v16 can support the older file formats as well, however + // it breaks the TestForceVersion unit test in scorch_test.go + // commenting the code below to avoid any breakages. + // // v16 and above are able to handle upgrade scenarios, so no need to load // the force load the older plugins - if defaultSegmentPlugin.Version() >= 16 { - return defaultSegmentPlugin, nil - } + // if defaultSegmentPlugin.Version() >= 16 { + // return defaultSegmentPlugin, nil + // } if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { diff --git a/index_test.go b/index_test.go index aeeca072e..561118317 100644 --- a/index_test.go +++ b/index_test.go @@ -504,7 +504,7 @@ func TestBytesRead(t *testing.T) { } } -func TestIndexSectionsNew(t *testing.T) { +func TestIndexSectionsUpgrade(t *testing.T) { tmpIndexPath := createTmpIndexPath(t) defer cleanupTmpIndexPath(t, tmpIndexPath) @@ -524,9 +524,15 @@ func TestIndexSectionsNew(t *testing.T) { typeFieldMapping := NewTextFieldMapping() typeFieldMapping.Store = false + typeFieldMapping.DocValues = true documentMapping.AddFieldMappingsAt("type", typeFieldMapping) - idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + config := map[string]interface{}{ + "forceSegmentVersion": 11, + "forceSegmentType": "zap", + } + + idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config) if err != nil { t.Fatal(err) } @@ -546,32 +552,19 @@ func TestIndexSectionsNew(t *testing.T) { if err != nil { t.Fatalf("failed to index batch %v\n", err) } + typeFacet := NewFacetRequest("type", 2) query := NewQueryStringQuery("united") - searchRequest := NewSearchRequestOptions(query, int(10), 0, true) - res, err := idx.Search(searchRequest) - if err != nil { - t.Error(err) - } - fmt.Printf("the results from the search %v", res) -} - -func TestIndexSectionsOpen(t *testing.T) { - idx, err := Open("/Users/thejasbhat/fts/data/beer-sample._default.bix_5c5ffb88b031ecd1_acbbef99.pindex") - if err != nil { - t.Fatal(err) - } - - typeFacet := NewFacetRequest("type", 2) - query := NewQueryStringQuery("wine") - searchRequest := NewSearchRequestOptions(query, int(10), 0, false) + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) searchRequest.AddFacet("types", typeFacet) res, err := idx.Search(searchRequest) if err != nil { t.Error(err) } - fmt.Printf("the search result for query %v is :\n %v\n", query, res) + if res.Facets["types"].Total != 9 { + t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total) + } } func getBatchFromData(idx Index, fileName string) (*Batch, error) {