modified test case

go.mod

@@ -18,11 +18,6 @@ require (
 	github.com/blevesearch/stempel v0.2.0
 	github.com/blevesearch/upsidedown_store_api v1.0.2
 	github.com/blevesearch/vellum v1.0.10
-	github.com/blevesearch/zapx/v11 v11.3.10
-	github.com/blevesearch/zapx/v12 v12.3.10
-	github.com/blevesearch/zapx/v13 v13.3.10
-	github.com/blevesearch/zapx/v14 v14.3.10
-	github.com/blevesearch/zapx/v15 v15.3.13
 	github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2

go.sum

@@ -33,18 +33,6 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
 github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
 github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
-github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
-github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
-github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
-github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
-github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
-github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
-github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
-github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
-github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80 h1:4zHuZvRJMo7GKqcbLXD1QpQdqQTqy5OIbUnNnZH6YzU=
-github.com/blevesearch/zapx/v16 v16.0.0-20231201163306-89e8af44fd80/go.mod h1:WIOW42mP6pIF9TyeUYG3u7KIhy0ml8Hnt3R5O925tjI=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

index/scorch/segment_plugin.go

@@ -109,6 +109,8 @@ func chooseSegmentPlugin(forcedSegmentType string,
 	// v16 and above are able to handle upgrade scenarios, so no need to load
 	// the force load the older plugins
 	if defaultSegmentPlugin.Version() >= 16 {
+		fmt.Printf("WARN: using v16 plugin to load index of segment "+
+			"type %v\n", forcedSegmentVersion)
 		return defaultSegmentPlugin, nil
 	}
 

index_test.go

@@ -15,6 +15,7 @@
 package bleve
 
 import (
+	"archive/zip"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -274,6 +275,72 @@ func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMappin
 	return statValError
 }
 
+func CopyFile(source string, dest string) (err error) {
+	sourcefile, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer sourcefile.Close()
+
+	destfile, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer destfile.Close()
+
+	_, err = io.Copy(destfile, sourcefile)
+	if err == nil {
+		sourceinfo, err := os.Stat(source)
+		if err != nil {
+			err = os.Chmod(dest, sourceinfo.Mode())
+		}
+	}
+	return
+}
+
+func CopyDir(source *zip.ReadCloser, dest string) (err error) {
+	// Extract the files from the zip
+	for _, f := range source.File {
+		// Create the destination file path
+		filePath := filepath.Join(dest, f.Name)
+
+		// Check if the file is a directory
+		if f.FileInfo().IsDir() {
+			// Create the directory
+			if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
+				panic(err)
+			}
+			continue
+		}
+
+		// Create the parent directory if it doesn't exist
+		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+			panic(err)
+		}
+
+		// Create an empty destination file
+		dstFile, err := os.OpenFile(filePath, os.O_WRONLY|
+			os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			panic(err)
+		}
+
+		// Open the file in the zip and copy its contents to the destination file
+		srcFile, err := f.Open()
+		if err != nil {
+			panic(err)
+		}
+		if _, err := io.Copy(dstFile, srcFile); err != nil {
+			panic(err)
+		}
+
+		// Close the files
+		dstFile.Close()
+		srcFile.Close()
+	}
+	return nil
+}
+
 func TestIndexSectionsUpgrade(t *testing.T) {
 	tmpIndexPath := createTmpIndexPath(t)
 	defer cleanupTmpIndexPath(t, tmpIndexPath)
@@ -297,21 +364,47 @@ func TestIndexSectionsUpgrade(t *testing.T) {
 	typeFieldMapping.DocValues = true
 	documentMapping.AddFieldMappingsAt("type", typeFieldMapping)
 
+	// create a v16 index
+	idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	config := map[string]interface{}{
 		"forceSegmentVersion": 11,
 		"forceSegmentType":    "zap",
 	}
 
-	idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config)
+	r, err := zip.OpenReader("./data/test/upgrade/bleve-testIdx-v11.zip")
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	err = CopyDir(r, "./data/test/upgrade/")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	idxV11Path := "./data/test/upgrade/bleve-testIdx-v11"
+	// this an index with old file format - v11. however, when its opened the
+	// back data structures are in accordance with v16 -> upgrade
+	idxV11, err := OpenUsing(idxV11Path, config)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer func() {
 		err := idx.Close()
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		err = idxV11.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cleanupTmpIndexPath(t, idxV11Path)
+
 	}()
 
 	batch, err := getBatchFromData(idx, "sample-data.json")
@@ -327,13 +420,20 @@ func TestIndexSectionsUpgrade(t *testing.T) {
 
 	searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
 	searchRequest.AddFacet("types", typeFacet)
+
 	res, err := idx.Search(searchRequest)
 	if err != nil {
 		t.Error(err)
 	}
 
-	if res.Facets["types"].Total != 9 {
-		t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total)
+	resV11, err := idxV11.Search(searchRequest)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if res.Facets["types"].Total != resV11.Facets["types"].Total {
+		t.Errorf("expected count for types facets is %v got %v",
+			resV11.Facets["types"].Total, res.Facets["types"].Total)
 	}
 }