diff --git a/data/test/upgrade/bleve-testIdx-v11.zip b/data/test/upgrade/bleve-testIdx-v11.zip new file mode 100644 index 000000000..e0d276ed2 Binary files /dev/null and b/data/test/upgrade/bleve-testIdx-v11.zip differ diff --git a/go.mod b/go.mod index 8a458b8a8..4b1ce1e77 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,6 @@ require ( github.com/blevesearch/stempel v0.2.0 github.com/blevesearch/upsidedown_store_api v1.0.2 github.com/blevesearch/vellum v1.0.10 - github.com/blevesearch/zapx/v11 v11.3.10 - github.com/blevesearch/zapx/v12 v12.3.10 - github.com/blevesearch/zapx/v13 v13.3.10 - github.com/blevesearch/zapx/v14 v14.3.10 - github.com/blevesearch/zapx/v15 v15.3.13 github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80 github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 diff --git a/go.sum b/go.sum index 8ce16226e..0b72bc6bb 100644 --- a/go.sum +++ b/go.sum @@ -33,18 +33,6 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= -github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= -github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= -github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= -github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= -github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= -github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= -github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= -github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80 h1:4zHuZvRJMo7GKqcbLXD1QpQdqQTqy5OIbUnNnZH6YzU= -github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80/go.mod h1:WIOW42mP6pIF9TyeUYG3u7KIhy0ml8Hnt3R5O925tjI= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index 6aece49bb..b0e3df98c 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -109,6 +109,8 @@ func chooseSegmentPlugin(forcedSegmentType string, // v16 and above are able to handle upgrade scenarios, so no need to load // the force load the older plugins if defaultSegmentPlugin.Version() >= 16 { + fmt.Printf("WARN: using v16 plugin to load index of segment "+ + "type %v\n", forcedSegmentVersion) return defaultSegmentPlugin, nil } diff --git a/index_test.go b/index_test.go index e3e360d63..a5e91ef89 100644 --- a/index_test.go +++ b/index_test.go @@ -15,6 +15,7 @@ package bleve import ( + "archive/zip" "context" "encoding/json" "fmt" @@ -274,6 +275,72 @@ func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMappin return statValError } +func CopyFile(source string, dest string) (err error) { + sourcefile, err := os.Open(source) + if err != nil { + return err + } + defer sourcefile.Close() + + destfile, err := os.Create(dest) + if err != nil { + return err + } + defer destfile.Close() + + _, err = io.Copy(destfile, sourcefile) + if err == nil { + sourceinfo, err := os.Stat(source) + if err != nil { + err = os.Chmod(dest, sourceinfo.Mode()) + } + } + return +} + +func CopyDir(source *zip.ReadCloser, dest string) (err error) { + // Extract the files from the zip + for _, f := range source.File { + // Create the destination file path + filePath := filepath.Join(dest, f.Name) + + // Check if the file is a directory + if f.FileInfo().IsDir() { + // Create the directory + if err := os.MkdirAll(filePath, os.ModePerm); err != nil { + panic(err) + } + continue + } + + // Create the parent directory if it doesn't exist + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + panic(err) + } + + // Create an empty destination file + dstFile, err := os.OpenFile(filePath, os.O_WRONLY| + os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + panic(err) + } + + // Open the file in the zip and copy its contents to the destination file + srcFile, err := f.Open() + if err != nil { + panic(err) + } + if _, err := io.Copy(dstFile, srcFile); err != nil { + panic(err) + } + + // Close the files + dstFile.Close() + srcFile.Close() + } + return nil +} + func TestIndexSectionsUpgrade(t *testing.T) { tmpIndexPath := createTmpIndexPath(t) defer cleanupTmpIndexPath(t, tmpIndexPath) @@ -297,21 +364,47 @@ func TestIndexSectionsUpgrade(t *testing.T) { typeFieldMapping.DocValues = true documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + // create a v16 index + idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ "forceSegmentVersion": 11, "forceSegmentType": "zap", } - idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config) + r, err := zip.OpenReader("./data/test/upgrade/bleve-testIdx-v11.zip") if err != nil { t.Fatal(err) } + err = CopyDir(r, "./data/test/upgrade/") + if err != nil { + t.Fatal(err) + } + + idxV11Path := "./data/test/upgrade/bleve-testIdx-v11" + // this an index with old file format - v11. however, when its opened the + // back data structures are in accordance with v16 -> upgrade + idxV11, err := OpenUsing(idxV11Path, config) + if err != nil { + t.Fatal(err) + } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } + + err = idxV11.Close() + if err != nil { + t.Fatal(err) + } + + cleanupTmpIndexPath(t, idxV11Path) + }() batch, err := getBatchFromData(idx, "sample-data.json") @@ -327,13 +420,20 @@ func TestIndexSectionsUpgrade(t *testing.T) { searchRequest := NewSearchRequestOptions(query, int(10), 0, true) searchRequest.AddFacet("types", typeFacet) + res, err := idx.Search(searchRequest) if err != nil { t.Error(err) } - if res.Facets["types"].Total != 9 { - t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total) + resV11, err := idxV11.Search(searchRequest) + if err != nil { + t.Error(err) + } + + if res.Facets["types"].Total != resV11.Facets["types"].Total { + t.Errorf("expected count for types facets is %v got %v", + resV11.Facets["types"].Total, res.Facets["types"].Total) } }