Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor sparse vector integration tests #72

Merged
merged 3 commits into from
Sep 10, 2024
Merged

Refactor sparse vector integration tests #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
40266a5
refactor createVectorsForUpsert into generateVectors and generateVect…
austin-denoble Sep 10, 2024
41ba7f7
remove extra integration test
austin-denoble Sep 10, 2024
df512a3
allow all data plane tests to share vectorIds on the IntegrationTests…
austin-denoble Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 33 additions & 3 deletions pinecone/index_connection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,21 @@ func (ts *IntegrationTests) TestDeleteVectorsById() {
ctx := context.Background()
err := ts.idxConn.DeleteVectorsById(ctx, ts.vectorIds)
assert.NoError(ts.T(), err)
ts.vectorIds = []string{}

_, err = ts.idxConn.UpsertVectors(ctx, createVectorsForUpsert())
vectors := GenerateVectors(5, ts.dimension, true)

_, err = ts.idxConn.UpsertVectors(ctx, vectors)
if err != nil {
log.Fatalf("Failed to upsert vectors in TestDeleteVectorsById test. Error: %v", err)
}

vectorIds := make([]string, len(vectors))
for i, v := range vectors {
vectorIds[i] = v.Id
}

ts.vectorIds = append(ts.vectorIds, vectorIds...)
}

func (ts *IntegrationTests) TestDeleteVectorsByFilter() {
Expand All @@ -84,23 +94,43 @@ func (ts *IntegrationTests) TestDeleteVectorsByFilter() {
} else {
assert.NoError(ts.T(), err)
}
ts.vectorIds = []string{}

vectors := GenerateVectors(5, ts.dimension, true)

_, err = ts.idxConn.UpsertVectors(ctx, createVectorsForUpsert())
_, err = ts.idxConn.UpsertVectors(ctx, vectors)
if err != nil {
log.Fatalf("Failed to upsert vectors in TestDeleteVectorsById test. Error: %v", err)
}

vectorIds := make([]string, len(vectors))
for i, v := range vectors {
vectorIds[i] = v.Id
}

ts.vectorIds = append(ts.vectorIds, vectorIds...)
}

func (ts *IntegrationTests) TestDeleteAllVectorsInNamespace() {
ctx := context.Background()
err := ts.idxConn.DeleteAllVectorsInNamespace(ctx)
assert.NoError(ts.T(), err)
ts.vectorIds = []string{}

vectors := GenerateVectors(5, ts.dimension, true)

_, err = ts.idxConn.UpsertVectors(ctx, createVectorsForUpsert())
_, err = ts.idxConn.UpsertVectors(ctx, vectors)
if err != nil {
log.Fatalf("Failed to upsert vectors in TestDeleteVectorsById test. Error: %v", err)
}

vectorIds := make([]string, len(vectors))
for i, v := range vectors {
vectorIds[i] = v.Id
}

ts.vectorIds = append(ts.vectorIds, vectorIds...)

}

func (ts *IntegrationTests) TestDescribeIndexStats() {
Expand Down
62 changes: 46 additions & 16 deletions pinecone/test_suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"math/rand"
"time"

"google.golang.org/protobuf/types/known/structpb"
Expand Down Expand Up @@ -46,14 +47,14 @@ func (ts *IntegrationTests) SetupSuite() {
ts.idxConn = idxConn

// Deterministically create vectors
vectors := createVectorsForUpsert()
vectors := GenerateVectors(10, ts.dimension, false)

// Set vector IDs
// Add vector ids to the suite
vectorIds := make([]string, len(vectors))
for i, v := range vectors {
vectorIds[i] = v.Id
}
ts.vectorIds = vectorIds
ts.vectorIds = append(ts.vectorIds, vectorIds...)

// Upsert vectors
err = upsertVectors(ts, ctx, vectors)
Expand Down Expand Up @@ -103,10 +104,17 @@ func upsertVectors(ts *IntegrationTests, ctx context.Context, vectors []*Vector)
_, err := WaitUntilIndexReady(ts, ctx)
require.NoError(ts.T(), err)

ids := make([]string, len(vectors))
for i, v := range vectors {
ids[i] = v.Id
}

upsertVectors, err := ts.idxConn.UpsertVectors(ctx, vectors)
require.NoError(ts.T(), err)
fmt.Printf("Upserted vectors: %v into host: %s\n", upsertVectors, ts.host)

ts.vectorIds = append(ts.vectorIds, ids...)

return nil
}

Expand Down Expand Up @@ -150,26 +158,48 @@ func WaitUntilIndexReady(ts *IntegrationTests, ctx context.Context) (bool, error
}
}

func createVectorsForUpsert() []*Vector {
vectors := make([]*Vector, 5)
for i := 0; i < 5; i++ {
func GenerateVectors(numOfVectors int, dimension int32, isSparse bool) []*Vector {
vectors := make([]*Vector, numOfVectors)

for i := 0; i < int(numOfVectors); i++ {
randomFloats := generateVectorValues(dimension)
vectors[i] = &Vector{
Id: fmt.Sprintf("vector-%d", i+1),
Values: []float32{float32(i), float32(i) + 0.1, float32(i) + 0.2, float32(i) + 0.3, float32(i) + 0.4},
SparseValues: &SparseValues{
Indices: []uint32{0, 1, 2, 3, 4},
Values: []float32{float32(i), float32(i) + 0.1, float32(i) + 0.2, float32(i) + 0.3, float32(i) + 0.4},
},
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"genre": {Kind: &structpb.Value_StringValue{StringValue: "classical"}},
},
Id: fmt.Sprintf("vector-%d", i),
Values: randomFloats,
}

if isSparse {
var sparseValues SparseValues
for j := 0; j < int(dimension); j++ {
sparseValues.Indices = append(sparseValues.Indices, uint32(j))
}
sparseValues.Values = generateVectorValues(dimension)
vectors[i].SparseValues = &sparseValues
}

metadata := &structpb.Struct{
Fields: map[string]*structpb.Value{
"genre": {Kind: &structpb.Value_StringValue{StringValue: "classical"}},
},
}
vectors[i].Metadata = metadata
}

return vectors
}

func generateVectorValues(dimension int32) []float32 {
maxInt := 1000000 // A large integer to normalize the float values
values := make([]float32, dimension)

for i := int32(0); i < dimension; i++ {
// Generate a random integer and normalize it to the range [0, 1)
values[i] = float32(rand.Intn(maxInt)) / float32(maxInt)
}

return values
}

func BuildServerlessTestIndex(in *Client, idxName string) *Index {
ctx := context.Background()

Expand Down