Skip to content

Commit

Permalink
reverting plugin changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Thejas-bhat
Thejas-bhat committed Dec 6, 2023
1 parent 8c0c405 commit a2642ff
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 206 deletions.
Binary file removed data/test/upgrade/bleve-testIdx-v11.zip
View file
Binary file not shown.
5 changes: 5 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ require (
github.com/blevesearch/stempel v0.2.0
github.com/blevesearch/upsidedown_store_api v1.0.2
github.com/blevesearch/vellum v1.0.10
github.com/blevesearch/zapx/v11 v11.3.10
github.com/blevesearch/zapx/v12 v12.3.10
github.com/blevesearch/zapx/v13 v13.3.10
github.com/blevesearch/zapx/v14 v14.3.10
github.com/blevesearch/zapx/v15 v15.3.13
github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80
github.com/couchbase/moss v0.2.0
github.com/golang/protobuf v1.3.2
Expand Down
10 changes: 10 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80 h1:4zHuZvRJMo7GKqcbLXD1QpQdqQTqy5OIbUnNnZH6YzU=
github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80/go.mod h1:WIOW42mP6pIF9TyeUYG3u7KIhy0ml8Hnt3R5O925tjI=
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
Expand Down
56 changes: 28 additions & 28 deletions index/scorch/scorch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2172,34 +2172,34 @@ func TestAllFieldWithDifferentTermVectorsEnabled(t *testing.T) {
}
}

func TestForceVersion(t *testing.T) {
cfg := map[string]interface{}{}
cfg["forceSegmentType"] = "zap"
cfg["forceSegmentVersion"] = 11
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewScorch(Name, cfg, analysisQueue)
if err != nil {
t.Fatalf("error opening a supported vesion: %v", err)
}
s := idx.(*Scorch)
if s.segPlugin.Version() != 11 {
t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 11, s.segPlugin.Version())
}
cfg["forceSegmentVersion"] = 12
idx, err = NewScorch(Name, cfg, analysisQueue)
if err != nil {
t.Fatalf("error opening a supported vesion: %v", err)
}
s = idx.(*Scorch)
if s.segPlugin.Version() != 12 {
t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 12, s.segPlugin.Version())
}
cfg["forceSegmentVersion"] = 10
idx, err = NewScorch(Name, cfg, analysisQueue)
if err == nil {
t.Fatalf("expected an error opening an unsupported vesion, got nil")
}
}
// func TestForceVersion(t *testing.T) {
// cfg := map[string]interface{}{}
// cfg["forceSegmentType"] = "zap"
// cfg["forceSegmentVersion"] = 11
// analysisQueue := index.NewAnalysisQueue(1)
// idx, err := NewScorch(Name, cfg, analysisQueue)
// if err != nil {
// t.Fatalf("error opening a supported vesion: %v", err)
// }
// s := idx.(*Scorch)
// if s.segPlugin.Version() != 11 {
// t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 11, s.segPlugin.Version())
// }
// cfg["forceSegmentVersion"] = 12
// idx, err = NewScorch(Name, cfg, analysisQueue)
// if err != nil {
// t.Fatalf("error opening a supported vesion: %v", err)
// }
// s = idx.(*Scorch)
// if s.segPlugin.Version() != 12 {
// t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 12, s.segPlugin.Version())
// }
// cfg["forceSegmentVersion"] = 10
// idx, err = NewScorch(Name, cfg, analysisQueue)
// if err == nil {
// t.Fatalf("expected an error opening an unsupported vesion, got nil")
// }
// }

func TestIndexForceMerge(t *testing.T) {
cfg := CreateConfig("TestIndexForceMerge")
Expand Down
24 changes: 11 additions & 13 deletions index/scorch/segment_plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ import (
"github.com/blevesearch/bleve/v2/geo"
segment "github.com/blevesearch/scorch_segment_api/v2"

zapv11 "github.com/blevesearch/zapx/v11"
zapv12 "github.com/blevesearch/zapx/v12"
zapv13 "github.com/blevesearch/zapx/v13"
zapv14 "github.com/blevesearch/zapx/v14"
zapv15 "github.com/blevesearch/zapx/v15"
zapv16 "github.com/blevesearch/zapx/v16"
)

Expand Down Expand Up @@ -69,7 +74,12 @@ var defaultSegmentPlugin SegmentPlugin

func init() {
ResetSegmentPlugins()
RegisterSegmentPlugin(&zapv16.ZapPlugin{}, true)
RegisterSegmentPlugin(&zapv16.ZapPlugin{}, false)
RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true)
RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false)
RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false)
RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false)
RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false)
}

func ResetSegmentPlugins() {
Expand Down Expand Up @@ -102,18 +112,6 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) {

func chooseSegmentPlugin(forcedSegmentType string,
forcedSegmentVersion uint32) (SegmentPlugin, error) {
// TBD: zap v16 can support the older file formats as well, however
// it breaks the TestForceVersion unit test in scorch_test.go
// commenting the code below to avoid any breakages.
//
// v16 and above are able to handle upgrade scenarios, so no need to load
// the force load the older plugins
if defaultSegmentPlugin.Version() >= 16 {
fmt.Printf("WARN: using v16 plugin to load index of segment "+
"type %v\n", forcedSegmentVersion)
return defaultSegmentPlugin, nil
}

if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {
return segPlugin, nil
Expand Down
165 changes: 0 additions & 165 deletions index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package bleve

import (
"archive/zip"
"context"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -275,169 +274,6 @@ func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMappin
return statValError
}

func CopyFile(source string, dest string) (err error) {
sourcefile, err := os.Open(source)
if err != nil {
return err
}
defer sourcefile.Close()

destfile, err := os.Create(dest)
if err != nil {
return err
}
defer destfile.Close()

_, err = io.Copy(destfile, sourcefile)
if err == nil {
sourceinfo, err := os.Stat(source)
if err != nil {
err = os.Chmod(dest, sourceinfo.Mode())
}
}
return
}

func CopyDir(source *zip.ReadCloser, dest string) (err error) {
// Extract the files from the zip
for _, f := range source.File {
// Create the destination file path
filePath := filepath.Join(dest, f.Name)

// Check if the file is a directory
if f.FileInfo().IsDir() {
// Create the directory
if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
panic(err)
}
continue
}

// Create the parent directory if it doesn't exist
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
panic(err)
}

// Create an empty destination file
dstFile, err := os.OpenFile(filePath, os.O_WRONLY|
os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
panic(err)
}

// Open the file in the zip and copy its contents to the destination file
srcFile, err := f.Open()
if err != nil {
panic(err)
}
if _, err := io.Copy(dstFile, srcFile); err != nil {
panic(err)
}

// Close the files
dstFile.Close()
srcFile.Close()
}
return nil
}

func TestIndexSectionsUpgrade(t *testing.T) {
tmpIndexPath := createTmpIndexPath(t)
defer cleanupTmpIndexPath(t, tmpIndexPath)

indexMapping := NewIndexMapping()
indexMapping.TypeField = "type"
indexMapping.DefaultAnalyzer = "en"
documentMapping := NewDocumentMapping()
indexMapping.AddDocumentMapping("hotel", documentMapping)
indexMapping.StoreDynamic = false
indexMapping.DocValuesDynamic = false
contentFieldMapping := NewTextFieldMapping()
contentFieldMapping.IncludeInAll = false

reviewsMapping := NewDocumentMapping()
reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping)
documentMapping.AddSubDocumentMapping("reviews", reviewsMapping)

typeFieldMapping := NewTextFieldMapping()
typeFieldMapping.Store = false
typeFieldMapping.DocValues = true
documentMapping.AddFieldMappingsAt("type", typeFieldMapping)

// create a v16 index
idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
if err != nil {
t.Fatal(err)
}

config := map[string]interface{}{
"forceSegmentVersion": 11,
"forceSegmentType": "zap",
}

r, err := zip.OpenReader("./data/test/upgrade/bleve-testIdx-v11.zip")
if err != nil {
t.Fatal(err)
}

err = CopyDir(r, "./data/test/upgrade/")
if err != nil {
t.Fatal(err)
}

idxV11Path := "./data/test/upgrade/bleve-testIdx-v11"
// this an index with old file format - v11. however, when its opened the
// back data structures are in accordance with v16 -> upgrade
idxV11, err := OpenUsing(idxV11Path, config)
if err != nil {
t.Fatal(err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}

err = idxV11.Close()
if err != nil {
t.Fatal(err)
}

cleanupTmpIndexPath(t, idxV11Path)

}()

batch, err := getBatchFromData(idx, "sample-data.json")
if err != nil {
t.Fatalf("failed to form a batch")
}
err = idx.Batch(batch)
if err != nil {
t.Fatalf("failed to index batch %v\n", err)
}
typeFacet := NewFacetRequest("type", 2)
query := NewQueryStringQuery("united")

searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
searchRequest.AddFacet("types", typeFacet)

res, err := idx.Search(searchRequest)
if err != nil {
t.Error(err)
}

resV11, err := idxV11.Search(searchRequest)
if err != nil {
t.Error(err)
}

if res.Facets["types"].Total != resV11.Facets["types"].Total {
t.Errorf("expected count for types facets is %v got %v",
resV11.Facets["types"].Total, res.Facets["types"].Total)
}
}

/*
func TestBytesWritten(t *testing.T) {
tmpIndexPath := createTmpIndexPath(t)

Expand Down Expand Up @@ -816,7 +652,6 @@ func TestBytesReadStored(t *testing.T) {
t.Fatalf("expected the bytes read stat to be around 42, got %v", bytesRead-prevBytesRead)
}
}
*/

func getBatchFromData(idx Index, fileName string) (*Batch, error) {
pwd, err := os.Getwd()
Expand Down

0 comments on commit a2642ff

Please sign in to comment.