From 4d94833b951b59354934d00f9b8cebe6297ee89e Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Mon, 16 Dec 2024 19:52:22 -0700 Subject: [PATCH 1/4] Fix interpreting scorch config: "fieldTFRCacheThreshold" + Firstly, refactor FieldTFRCacheThreshold to fieldTFRCacheThreshold for _some_ naming consistency here. + This threshold can be used to toggle recycling of TermFieldReaders on/off. + Couchbase users will have the ability to provide this setting within a JSON payload, which when interpreted into a map[string]interface{} will need to be interpreted as a `float64`. + Should library users set it as `int` we'll honor that setting as well. --- index/scorch/snapshot_index.go | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 79840a41f..3e26930f5 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -52,15 +52,6 @@ type asynchSegmentResult struct { var reflectStaticSizeIndexSnapshot int -// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for -// a field in an index snapshot. Without this limit, when recycling TFRs, it is -// possible that a very large number of TFRs may be added to the recycle -// cache, which could eventually lead to significant memory consumption. -// This threshold can be overwritten by users at the library level by changing the -// exported variable, or at the index level by setting the FieldTFRCacheThreshold -// in the kvConfig. -var DefaultFieldTFRCacheThreshold uint64 = 10 - func init() { var is interface{} = IndexSnapshot{} reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) @@ -640,13 +631,30 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap } } -func (is *IndexSnapshot) getFieldTFRCacheThreshold() uint64 { +// defaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for +// a field in an index snapshot. Without this limit, when recycling TFRs, it is +// possible that a very large number of TFRs may be added to the recycle +// cache, which could eventually lead to significant memory consumption. +// This threshold can be overwritten by users at the library level by changing the +// exported variable, or at the index level by setting the "fieldTFRCacheThreshold" +// in the kvConfig. +const defaultFieldTFRCacheThreshold int = 10 + +func (is *IndexSnapshot) getFieldTFRCacheThreshold() int { if is.parent.config != nil { - if _, ok := is.parent.config["FieldTFRCacheThreshold"]; ok { - return is.parent.config["FieldTFRCacheThreshold"].(uint64) + if _, exists := is.parent.config["fieldTFRCacheThreshold"]; exists { + val := is.parent.config["fieldTFRCacheThreshold"] + if x, ok := val.(float64); ok { + // JSON unmarshal-ed into a map[string]interface{} will default + // to float64 for numbers, so we need to check for float64 first. + return int(x) + } else if x, ok := val.(int); ok { + // If library users provided an int in the config, we'll honor it. + return x + } } } - return DefaultFieldTFRCacheThreshold + return defaultFieldTFRCacheThreshold } func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { @@ -670,7 +678,7 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade if is.fieldTFRs == nil { is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{} } - if uint64(len(is.fieldTFRs[tfr.field])) < is.getFieldTFRCacheThreshold() { + if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() { tfr.bytesRead = 0 is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr) } From 51dc4532b0911c37829c845dd9e8211206197dca Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Tue, 17 Dec 2024 08:41:31 -0700 Subject: [PATCH 2/4] Disabling fieldTFRCacheThreshold by default --- index/scorch/snapshot_index.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 3e26930f5..2ae7573f4 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -638,7 +638,7 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap // This threshold can be overwritten by users at the library level by changing the // exported variable, or at the index level by setting the "fieldTFRCacheThreshold" // in the kvConfig. -const defaultFieldTFRCacheThreshold int = 10 +var DefaultFieldTFRCacheThreshold int = 0 // disabled because it causes MB-64604 func (is *IndexSnapshot) getFieldTFRCacheThreshold() int { if is.parent.config != nil { @@ -654,7 +654,7 @@ func (is *IndexSnapshot) getFieldTFRCacheThreshold() int { } } } - return defaultFieldTFRCacheThreshold + return DefaultFieldTFRCacheThreshold } func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { From 1da8476797b649e2b47caeaef0ca7dd37af0571f Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Tue, 17 Dec 2024 09:02:28 -0700 Subject: [PATCH 3/4] Update bytesRead tests to reflect change in TFR behavior --- index_test.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/index_test.go b/index_test.go index e5094440d..a857d5ecb 100644 --- a/index_test.go +++ b/index_test.go @@ -421,8 +421,8 @@ func TestBytesRead(t *testing.T) { } stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 23 && res.Cost == bytesRead-prevBytesRead { - t.Fatalf("expected bytes read for query string 23, got %v", + if bytesRead-prevBytesRead != 66 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for query string 66, got %v", bytesRead-prevBytesRead) } prevBytesRead = bytesRead @@ -454,8 +454,8 @@ func TestBytesRead(t *testing.T) { stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if !approxSame(bytesRead-prevBytesRead, 150) && res.Cost == bytesRead-prevBytesRead { - t.Fatalf("expected bytes read for faceted query is around 150, got %v", + if !approxSame(bytesRead-prevBytesRead, 196) && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for faceted query is around 196, got %v", bytesRead-prevBytesRead) } prevBytesRead = bytesRead @@ -487,8 +487,8 @@ func TestBytesRead(t *testing.T) { stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 60 && res.Cost == bytesRead-prevBytesRead { - t.Fatalf("expected bytes read for query with highlighter is 60, got %v", + if bytesRead-prevBytesRead != 105 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for query with highlighter is 105, got %v", bytesRead-prevBytesRead) } prevBytesRead = bytesRead @@ -504,8 +504,8 @@ func TestBytesRead(t *testing.T) { // since it's created afresh and not reused stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 83 && res.Cost == bytesRead-prevBytesRead { - t.Fatalf("expected bytes read for disjunction query is 83, got %v", + if bytesRead-prevBytesRead != 120 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for disjunction query is 120, got %v", bytesRead-prevBytesRead) } } @@ -577,8 +577,8 @@ func TestBytesReadStored(t *testing.T) { } stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 15 && bytesRead-prevBytesRead == res.Cost { - t.Fatalf("expected the bytes read stat to be around 15, got %v", bytesRead-prevBytesRead) + if bytesRead-prevBytesRead != 48 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 48, got %v", bytesRead-prevBytesRead) } prevBytesRead = bytesRead @@ -592,8 +592,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ = idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 26478 && bytesRead-prevBytesRead == res.Cost { - t.Fatalf("expected the bytes read stat to be around 26478, got %v", + if bytesRead-prevBytesRead != 26511 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 26511, got %v", bytesRead-prevBytesRead) } idx.Close() @@ -653,8 +653,8 @@ func TestBytesReadStored(t *testing.T) { } stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 12 && bytesRead-prevBytesRead == res.Cost { - t.Fatalf("expected the bytes read stat to be around 12, got %v", bytesRead-prevBytesRead) + if bytesRead-prevBytesRead != 47 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 47, got %v", bytesRead-prevBytesRead) } prevBytesRead = bytesRead @@ -666,8 +666,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead-prevBytesRead != 42 && bytesRead-prevBytesRead == res.Cost { - t.Fatalf("expected the bytes read stat to be around 42, got %v", bytesRead-prevBytesRead) + if bytesRead-prevBytesRead != 77 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 77, got %v", bytesRead-prevBytesRead) } } From f344f43d0a579a48016908db4898d876a7a8bc39 Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Tue, 17 Dec 2024 10:20:03 -0700 Subject: [PATCH 4/4] Fix typo in comment --- index/scorch/snapshot_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 2ae7573f4..8ad073f0f 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -631,7 +631,7 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap } } -// defaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for +// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for // a field in an index snapshot. Without this limit, when recycling TFRs, it is // possible that a very large number of TFRs may be added to the recycle // cache, which could eventually lead to significant memory consumption.