wip: making v16 as default zap plugin

go.mod

@@ -18,11 +18,6 @@ require (
 	github.com/blevesearch/stempel v0.2.0
 	github.com/blevesearch/upsidedown_store_api v1.0.2
 	github.com/blevesearch/vellum v1.0.10
-	github.com/blevesearch/zapx/v11 v11.3.10
-	github.com/blevesearch/zapx/v12 v12.3.10
-	github.com/blevesearch/zapx/v13 v13.3.10
-	github.com/blevesearch/zapx/v14 v14.3.10
-	github.com/blevesearch/zapx/v15 v15.3.13
 	github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
@@ -44,3 +39,5 @@ require (
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/sys v0.13.0 // indirect
 )
+
+replace github.com/blevesearch/zapx/v16 => ../zapx

go.sum

@@ -33,18 +33,6 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
 github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
 github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
-github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
-github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
-github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
-github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
-github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
-github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
-github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
-github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
-github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33 h1:wHyGpG9ZPs4QmVAEg14Eg6vknP1qub9tjGpQoL5MCDc=
-github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33/go.mod h1:1wZRO5mWgYFa9LDNomxqbcyJ1JU827X/V7fCIdssSIg=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

index/scorch/segment_plugin.go

@@ -23,11 +23,6 @@ import (
 	"github.com/blevesearch/bleve/v2/geo"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 
-	zapv11 "github.com/blevesearch/zapx/v11"
-	zapv12 "github.com/blevesearch/zapx/v12"
-	zapv13 "github.com/blevesearch/zapx/v13"
-	zapv14 "github.com/blevesearch/zapx/v14"
-	zapv15 "github.com/blevesearch/zapx/v15"
 	zapv16 "github.com/blevesearch/zapx/v16"
 )
 
@@ -75,11 +70,6 @@ var defaultSegmentPlugin SegmentPlugin
 func init() {
 	ResetSegmentPlugins()
 	RegisterSegmentPlugin(&zapv16.ZapPlugin{}, true)
-	RegisterSegmentPlugin(&zapv15.ZapPlugin{}, false)
-	RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false)
-	RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false)
-	RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false)
-	RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false)
 }
 
 func ResetSegmentPlugins() {
@@ -112,6 +102,16 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) {
 
 func chooseSegmentPlugin(forcedSegmentType string,
 	forcedSegmentVersion uint32) (SegmentPlugin, error) {
+	// TBD: zap v16 can support the older file formats as well, however
+	// it breaks the TestForceVersion unit test in scorch_test.go
+	// commenting the code below to avoid any breakages.
+	//
+	// v16 and above are able to handle upgrade scenarios, so no need to load
+	// the force load the older plugins
+	if defaultSegmentPlugin.Version() >= 16 {
+		return defaultSegmentPlugin, nil
+	}
+
 	if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {
 		if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {
 			return segPlugin, nil

index_test.go

@@ -274,6 +274,69 @@ func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMappin
 	return statValError
 }
 
+func TestIndexSectionsUpgrade(t *testing.T) {
+	tmpIndexPath := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPath)
+
+	indexMapping := NewIndexMapping()
+	indexMapping.TypeField = "type"
+	indexMapping.DefaultAnalyzer = "en"
+	documentMapping := NewDocumentMapping()
+	indexMapping.AddDocumentMapping("hotel", documentMapping)
+	indexMapping.StoreDynamic = false
+	indexMapping.DocValuesDynamic = false
+	contentFieldMapping := NewTextFieldMapping()
+	contentFieldMapping.IncludeInAll = false
+
+	reviewsMapping := NewDocumentMapping()
+	reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping)
+	documentMapping.AddSubDocumentMapping("reviews", reviewsMapping)
+
+	typeFieldMapping := NewTextFieldMapping()
+	typeFieldMapping.Store = false
+	typeFieldMapping.DocValues = true
+	documentMapping.AddFieldMappingsAt("type", typeFieldMapping)
+
+	config := map[string]interface{}{
+		"forceSegmentVersion": 11,
+		"forceSegmentType":    "zap",
+	}
+
+	idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	batch, err := getBatchFromData(idx, "sample-data.json")
+	if err != nil {
+		t.Fatalf("failed to form a batch")
+	}
+	err = idx.Batch(batch)
+	if err != nil {
+		t.Fatalf("failed to index batch %v\n", err)
+	}
+	typeFacet := NewFacetRequest("type", 2)
+	query := NewQueryStringQuery("united")
+
+	searchRequest := NewSearchRequestOptions(query, int(10), 0, true)
+	searchRequest.AddFacet("types", typeFacet)
+	res, err := idx.Search(searchRequest)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if res.Facets["types"].Total != 9 {
+		t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total)
+	}
+}
+
 func TestBytesWritten(t *testing.T) {
 	tmpIndexPath := createTmpIndexPath(t)
 
@@ -401,8 +464,8 @@ func TestBytesRead(t *testing.T) {
 	}
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if prevBytesRead != 21631 && res.Cost == prevBytesRead {
-		t.Fatalf("expected bytes read for query string 21631, got %v",
+	if prevBytesRead != 21639 && res.Cost == prevBytesRead {
+		t.Fatalf("expected bytes read for query string 21639, got %v",
 			prevBytesRead)
 	}
 
@@ -580,8 +643,8 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 11493 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 11493, got %v", bytesRead)
+	if bytesRead != 11501 && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around 11501, got %v", bytesRead)
 	}
 	prevBytesRead := bytesRead
 
@@ -651,8 +714,8 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ = idx1.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 3679 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 3679, got %v", bytesRead)
+	if bytesRead != 3687 && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around 3687, got %v", bytesRead)
 	}
 	prevBytesRead = bytesRead