Skip to content

Commit

Permalink
add index optimization type in mapping
Browse files Browse the repository at this point in the history
  • Loading branch information
metonymic-smokey committed Jan 15, 2024
1 parent 79c327f commit 348d132
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 16 deletions.
37 changes: 22 additions & 15 deletions document/field_vector.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,13 @@ func init() {
const DefaultVectorIndexingOptions = index.IndexField

type VectorField struct {
name string
dims int // Dimensionality of the vector
similarity string // Similarity metric to use for scoring
options index.FieldIndexingOptions
value []float32
numPlainTextBytes uint64
name string
dims int // Dimensionality of the vector
similarity string // Similarity metric to use for scoring
options index.FieldIndexingOptions
value []float32
numPlainTextBytes uint64
vectorIndexOptimizedFor string // Optimization applied to this index.
}

func (n *VectorField) Size() int {
Expand Down Expand Up @@ -95,25 +96,27 @@ func (n *VectorField) GoString() string {
// For the sake of not polluting the API, we are keeping arrayPositions as a
// parameter, but it is not used.
func NewVectorField(name string, arrayPositions []uint64,
vector []float32, dims int, similarity string) *VectorField {
vector []float32, dims int, similarity, vectorIndexOptimizedFor string) *VectorField {
return NewVectorFieldWithIndexingOptions(name, arrayPositions,
vector, dims, similarity, DefaultVectorIndexingOptions)
vector, dims, similarity, vectorIndexOptimizedFor,
DefaultVectorIndexingOptions)
}

// For the sake of not polluting the API, we are keeping arrayPositions as a
// parameter, but it is not used.
func NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64,
vector []float32, dims int, similarity string,
vector []float32, dims int, similarity, vectorIndexOptimizedFor string,
options index.FieldIndexingOptions) *VectorField {
options = options | DefaultVectorIndexingOptions

return &VectorField{
name: name,
dims: dims,
similarity: similarity,
options: options,
value: vector,
numPlainTextBytes: numBytesFloat32s(vector),
name: name,
dims: dims,
similarity: similarity,
options: options,
value: vector,
numPlainTextBytes: numBytesFloat32s(vector),
vectorIndexOptimizedFor: vectorIndexOptimizedFor,
}
}

Expand All @@ -136,3 +139,7 @@ func (n *VectorField) Dims() int {
func (n *VectorField) Similarity() string {
return n.similarity
}

func (n *VectorField) IndexOptimizedFor() string {
return n.vectorIndexOptimizedFor
}
8 changes: 8 additions & 0 deletions mapping/field.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ type FieldMapping struct {
// vector fields.
// See: index.DefaultSimilarityMetric & index.SupportedSimilarityMetrics
Similarity string `json:"similarity,omitempty"`

// The type of index based on the parameter to optimize for.
VectorIndexOptimizedFor string `json:"vector_index_optimized_for,omitempty"`
}

// NewTextFieldMapping returns a default field mapping for text
Expand Down Expand Up @@ -466,6 +469,11 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error {
if err != nil {
return err
}
case "vector_index_optimized_for":
err := json.Unmarshal(v, &fm.VectorIndexOptimizedFor)
if err != nil {
return err
}
default:
invalidKeys = append(invalidKeys, k)
}
Expand Down
14 changes: 13 additions & 1 deletion mapping/mapping_vectors.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
fieldName := getFieldName(pathString, path, fm)
options := fm.Options()
field := document.NewVectorFieldWithIndexingOptions(fieldName,
indexes, vector, fm.Dims, fm.Similarity, options)
indexes, vector, fm.Dims, fm.Similarity, fm.VectorIndexOptimizedFor,
options)
context.doc.AddField(field)

// "_all" composite field is not applicable for vector field
Expand Down Expand Up @@ -162,6 +163,11 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,
field.Similarity = index.DefaultSimilarityMetric
}

if field.VectorIndexOptimizedFor == "" {
fmt.Printf("using the default optimization type \n")
field.VectorIndexOptimizedFor = index.DefaultIndexOptimizedFor
}

// following fields are not applicable for vector
// thus, we set them to default values
field.IncludeInAll = false
Expand Down Expand Up @@ -202,6 +208,12 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,
reflect.ValueOf(index.SupportedSimilarityMetrics).MapKeys())
}

if _, ok := index.SupportedVectorIndexOptimizations[field.VectorIndexOptimizedFor]; !ok {
return fmt.Errorf("field: '%s', invalid optimization "+
"metric: '%s', valid metrics are: %+v", field.Name, field.VectorIndexOptimizedFor,
reflect.ValueOf(index.SupportedVectorIndexOptimizations).MapKeys())
}

if fieldAliasCtx != nil { // writing to a nil map is unsafe
fieldAliasCtx[field.Name] = field
}
Expand Down

0 comments on commit 348d132

Please sign in to comment.