diff --git a/go.mod b/go.mod index 458eaf3a6..33ef6c469 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,6 @@ require ( github.com/blevesearch/stempel v0.2.0 github.com/blevesearch/upsidedown_store_api v1.0.2 github.com/blevesearch/vellum v1.0.10 - github.com/blevesearch/zapx/v11 v11.3.10 - github.com/blevesearch/zapx/v12 v12.3.10 - github.com/blevesearch/zapx/v13 v13.3.10 - github.com/blevesearch/zapx/v14 v14.3.10 - github.com/blevesearch/zapx/v15 v15.3.13 github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33 github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 @@ -44,3 +39,5 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/sys v0.13.0 // indirect ) + +replace github.com/blevesearch/zapx/v16 => ../zapx diff --git a/go.sum b/go.sum index 70e5fa405..70b36a321 100644 --- a/go.sum +++ b/go.sum @@ -33,18 +33,6 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= -github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= -github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= -github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= -github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= -github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= -github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= -github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= -github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33 h1:wHyGpG9ZPs4QmVAEg14Eg6vknP1qub9tjGpQoL5MCDc= -github.com/blevesearch/zapx/v16 v16.0.0-20231122165630-322e3d773a33/go.mod h1:1wZRO5mWgYFa9LDNomxqbcyJ1JU827X/V7fCIdssSIg= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index b3b9ba01f..6aece49bb 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -23,11 +23,6 @@ import ( "github.com/blevesearch/bleve/v2/geo" segment "github.com/blevesearch/scorch_segment_api/v2" - zapv11 "github.com/blevesearch/zapx/v11" - zapv12 "github.com/blevesearch/zapx/v12" - zapv13 "github.com/blevesearch/zapx/v13" - zapv14 "github.com/blevesearch/zapx/v14" - zapv15 "github.com/blevesearch/zapx/v15" zapv16 "github.com/blevesearch/zapx/v16" ) @@ -75,11 +70,6 @@ var defaultSegmentPlugin SegmentPlugin func init() { ResetSegmentPlugins() RegisterSegmentPlugin(&zapv16.ZapPlugin{}, true) - RegisterSegmentPlugin(&zapv15.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) - RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) } func ResetSegmentPlugins() { @@ -112,6 +102,16 @@ func SupportedSegmentTypeVersions(typ string) (rv []uint32) { func chooseSegmentPlugin(forcedSegmentType string, forcedSegmentVersion uint32) (SegmentPlugin, error) { + // TBD: zap v16 can support the older file formats as well, however + // it breaks the TestForceVersion unit test in scorch_test.go + // commenting the code below to avoid any breakages. + // + // v16 and above are able to handle upgrade scenarios, so no need to load + // the force load the older plugins + if defaultSegmentPlugin.Version() >= 16 { + return defaultSegmentPlugin, nil + } + if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { return segPlugin, nil diff --git a/index_test.go b/index_test.go index fbff77389..9f0005d77 100644 --- a/index_test.go +++ b/index_test.go @@ -274,6 +274,69 @@ func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMappin return statValError } +func TestIndexSectionsUpgrade(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("hotel", documentMapping) + indexMapping.StoreDynamic = false + indexMapping.DocValuesDynamic = false + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.IncludeInAll = false + + reviewsMapping := NewDocumentMapping() + reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping) + documentMapping.AddSubDocumentMapping("reviews", reviewsMapping) + + typeFieldMapping := NewTextFieldMapping() + typeFieldMapping.Store = false + typeFieldMapping.DocValues = true + documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + + config := map[string]interface{}{ + "forceSegmentVersion": 11, + "forceSegmentType": "zap", + } + + idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, config) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch, err := getBatchFromData(idx, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch") + } + err = idx.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + typeFacet := NewFacetRequest("type", 2) + query := NewQueryStringQuery("united") + + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) + searchRequest.AddFacet("types", typeFacet) + res, err := idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + if res.Facets["types"].Total != 9 { + t.Errorf("expected count for types facets is 9 got %v", res.Facets["types"].Total) + } +} + func TestBytesWritten(t *testing.T) { tmpIndexPath := createTmpIndexPath(t) @@ -401,7 +464,7 @@ func TestBytesRead(t *testing.T) { } stats, _ := idx.StatsMap()["index"].(map[string]interface{}) prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if prevBytesRead != 21631 && res.Cost == prevBytesRead { + if prevBytesRead != 21639 && res.Cost == prevBytesRead { t.Fatalf("expected bytes read for query string 21631, got %v", prevBytesRead) } @@ -580,7 +643,7 @@ func TestBytesReadStored(t *testing.T) { stats, _ := idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 11493 && bytesRead == res.Cost { + if bytesRead != 11501 && bytesRead == res.Cost { t.Fatalf("expected the bytes read stat to be around 11493, got %v", bytesRead) } prevBytesRead := bytesRead @@ -651,7 +714,7 @@ func TestBytesReadStored(t *testing.T) { stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 3679 && bytesRead == res.Cost { + if bytesRead != 3687 && bytesRead == res.Cost { t.Fatalf("expected the bytes read stat to be around 3679, got %v", bytesRead) } prevBytesRead = bytesRead