Skip to content

Commit

Permalink
code cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@Thejas-bhat
Thejas-bhat committed Nov 3, 2023
1 parent 9b32e4f commit f321aad
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 35 deletions.
12 changes: 7 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ go 1.19
require (
github.com/RoaringBitmap/roaring v1.2.3
github.com/bits-and-blooms/bitset v1.2.0
github.com/blevesearch/bleve_index_api v1.0.6
github.com/blevesearch/bleve_index_api v1.1.2
github.com/blevesearch/geo v0.1.18
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/blevesearch/go-porterstemmer v1.0.3
github.com/blevesearch/goleveldb v1.0.1
github.com/blevesearch/gtreap v0.1.1
github.com/blevesearch/scorch_segment_api/v2 v2.1.6
github.com/blevesearch/scorch_segment_api/v2 v2.2.2
github.com/blevesearch/segment v0.9.1
github.com/blevesearch/snowball v0.6.1
github.com/blevesearch/snowballstem v0.9.0
Expand All @@ -23,6 +23,7 @@ require (
github.com/blevesearch/zapx/v13 v13.3.10
github.com/blevesearch/zapx/v14 v14.3.10
github.com/blevesearch/zapx/v15 v15.3.13
github.com/blevesearch/zapx/v16 v16.0.0-00010101000000-000000000000
github.com/couchbase/moss v0.2.0
github.com/golang/protobuf v1.3.2
github.com/spf13/cobra v1.7.0
Expand All @@ -31,16 +32,17 @@ require (
)

require (
github.com/blevesearch/go-faiss v1.0.1 // indirect
github.com/blevesearch/mmap-go v1.0.4 // indirect
github.com/blevesearch/zapx v0.0.0-00010101000000-000000000000 // indirect
github.com/couchbase/ghistogram v0.1.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/sys v0.13.0 // indirect
)

replace github.com/blevesearch/zapx => ../zapx
replace github.com/blevesearch/zapx/v16 => ../zapx
16 changes: 10 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8=
github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
github.com/blevesearch/bleve_index_api v1.1.2 h1:A8MhXiNbZ9DI+lZytWkOY75MwesRdOlE+7/MzdC9YXY=
github.com/blevesearch/bleve_index_api v1.1.2/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
github.com/blevesearch/go-faiss v1.0.1 h1:B0/FGdmcdxHIM0DRPyy4aWk0ZjMTFbCsmIzra77GAxE=
github.com/blevesearch/go-faiss v1.0.1/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
Expand All @@ -17,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA=
github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
github.com/blevesearch/scorch_segment_api/v2 v2.2.2 h1:wRkJ2tAP0HsS4M2pK/qDkNV0PvUo33umcsjzLIgvVA4=
github.com/blevesearch/scorch_segment_api/v2 v2.2.2/go.mod h1:7mKEerrxzvfImS2pMvpfV5MGFYVcQ9NBTkBd/ApU7cI=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
Expand Down Expand Up @@ -83,14 +85,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
Expand Down
12 changes: 8 additions & 4 deletions index/scorch/segment_plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ import (
"github.com/blevesearch/bleve/v2/geo"
segment "github.com/blevesearch/scorch_segment_api/v2"

zapv16 "github.com/blevesearch/zapx"
zapv11 "github.com/blevesearch/zapx/v11"
zapv12 "github.com/blevesearch/zapx/v12"
zapv13 "github.com/blevesearch/zapx/v13"
zapv14 "github.com/blevesearch/zapx/v14"
zapv15 "github.com/blevesearch/zapx/v15"
zapv16 "github.com/blevesearch/zapx/v16"

Check failure on line 31 in index/scorch/segment_plugin.go

View workflow job for this annotation

GitHub Actions / test (1.18.x, ubuntu-latest) 

github.com/blevesearch/zapx/[email protected]: replacement directory ../zapx does not exist

Check failure on line 31 in index/scorch/segment_plugin.go

View workflow job for this annotation

GitHub Actions / test (1.19.x, ubuntu-latest) 

github.com/blevesearch/zapx/[email protected]: replacement directory ../zapx does not exist

Check failure on line 31 in index/scorch/segment_plugin.go

View workflow job for this annotation

GitHub Actions / test (1.19.x, macos-latest) 

github.com/blevesearch/zapx/[email protected]: replacement directory ../zapx does not exist

Check failure on line 31 in index/scorch/segment_plugin.go

View workflow job for this annotation

GitHub Actions / test (1.20.x, ubuntu-latest) 

github.com/blevesearch/zapx/[email protected]: replacement directory ../zapx does not exist
)

// SegmentPlugin represents the essential functions required by a package to plug in
Expand Down Expand Up @@ -113,11 +113,15 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) {
func chooseSegmentPlugin(forcedSegmentType string,
forcedSegmentVersion uint32) (SegmentPlugin, error) {

// TBD: zap v16 can support the older file formats as well, however
// it breaks the TestForceVersion unit test in scorch_test.go
// commenting the code below to avoid any breakages.
//
// v16 and above are able to handle upgrade scenarios, so no need to load
// the force load the older plugins
if defaultSegmentPlugin.Version() >= 16 {
return defaultSegmentPlugin, nil
}
// if defaultSegmentPlugin.Version() >= 16 {
// return defaultSegmentPlugin, nil
// }

if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {
Expand Down
33 changes: 13 additions & 20 deletions index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ func TestBytesRead(t *testing.T) {
}
}

func TestIndexSectionsNew(t *testing.T) {
func TestIndexSectionsUpgrade(t *testing.T) {
tmpIndexPath := createTmpIndexPath(t)
defer cleanupTmpIndexPath(t, tmpIndexPath)

Expand All @@ -524,9 +524,15 @@ func TestIndexSectionsNew(t *testing.T) {

typeFieldMapping := NewTextFieldMapping()
typeFieldMapping.Store = false
typeFieldMapping.DocValues = true
documentMapping.AddFieldMappingsAt("type", typeFieldMapping)

idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
config := map[string]interface{}{
"forceSegmentVersion": 11,
"forceSegmentType": "zap",
}

idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config)
if err != nil {
t.Fatal(err)
}
Expand All @@ -546,32 +552,19 @@ func TestIndexSectionsNew(t *testing.T) {
if err != nil {
t.Fatalf("failed to index batch %v\n", err)
}
typeFacet := NewFacetRequest("type", 2)
query := NewQueryStringQuery("united")
searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
res, err := idx.Search(searchRequest)
if err != nil {
t.Error(err)
}

fmt.Printf("the results from the search %v", res)
}

func TestIndexSectionsOpen(t *testing.T) {
idx, err := Open("/Users/thejasbhat/fts/data/beer-sample._default.bix_5c5ffb88b031ecd1_acbbef99.pindex")
if err != nil {
t.Fatal(err)
}

typeFacet := NewFacetRequest("type", 2)
query := NewQueryStringQuery("wine")
searchRequest := NewSearchRequestOptions(query, int(10), 0, false)
searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
searchRequest.AddFacet("types", typeFacet)
res, err := idx.Search(searchRequest)
if err != nil {
t.Error(err)
}

fmt.Printf("the search result for query %v is :\n %v\n", query, res)
if res.Facets["types"].Total != 9 {
t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total)
}
}

func getBatchFromData(idx Index, fileName string) (*Batch, error) {
Expand Down

0 comments on commit f321aad

Please sign in to comment.