Merge branch 'main' into users/markwallace/enable_inmemorytextsearcht…

…ests
microsoft · Oct 15, 2024 · 1e55feb · 1e55feb
2 parents af81b4a + afef598
commit 1e55feb
Show file tree

Hide file tree

Showing 19 changed files with 49 additions and 28 deletions.
diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
@@ -40,7 +40,7 @@
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
     <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
-    <PackageVersion Include="OllamaSharp" Version="3.0.10" />
+    <PackageVersion Include="OllamaSharp" Version="3.0.12" />
     <!-- Tokenizers -->
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
     <PackageVersion Include="Microsoft.DeepDev.TokenizerLib" Version="1.3.3" />
@@ -69,7 +69,7 @@
     <PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
     <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="8.0.0" />
     <PackageVersion Include="Verify.Xunit" Version="23.5.2" />
-    <PackageVersion Include="xunit" Version="2.9.0" />
+    <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.abstractions" Version="2.0.3" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageVersion Include="xretry" Version="1.9.0" />

diff --git a/dotnet/samples/GettingStarted/README.md b/dotnet/samples/GettingStarted/README.md
@@ -8,7 +8,7 @@ The examples can be run as integration tests but their code can also be copied t
 
 Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
 Bing and other resources. We suggest using .NET
-[Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets)
+[Secret Manager](https://learn.microsoft.com/aspnet/core/security/app-secrets)
 to avoid the risk of leaking secrets into the repository, branches and pull requests.
 You can also use environment variables if you prefer.
 

diff --git a/dotnet/samples/GettingStartedWithTextSearch/README.md b/dotnet/samples/GettingStartedWithTextSearch/README.md
@@ -8,10 +8,13 @@ The examples can be run as integration tests but their code can also be copied t
 
 Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
 Bing and other resources. We suggest using .NET
-[Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets)
+[Secret Manager](https://learn.microsoft.com/aspnet/core/security/app-secrets)
 to avoid the risk of leaking secrets into the repository, branches and pull requests.
 You can also use environment variables if you prefer.
 
+**NOTE**
+The `Step2_Search_For_RAG.RagWithBingTextSearchUsingFullPagesAsync` sample requires a large context window so we recommend using `gpt-4o` or `gpt-4o-mini` models.
+
 To set your secrets with Secret Manager:
 
 ```
@@ -23,6 +26,10 @@ dotnet user-secrets set "OpenAI:EmbeddingModelId" "..."
 dotnet user-secrets set "OpenAI:ChatModelId" "..."
 dotnet user-secrets set "OpenAI:ApiKey" "..."
 
+dotnet user-secrets set "Bing:ApiKey" "..."
+
+dotnet user-secrets set "Google:SearchEngineId" "..."
+dotnet user-secrets set "Google:ApiKey" "..."
 ```
 
 To set your secrets with environment variables, use these names:
@@ -31,4 +38,9 @@ To set your secrets with environment variables, use these names:
 OpenAI__EmbeddingModelId
 OpenAI__ChatModelId
 OpenAI__ApiKey
+
+Bing__ApiKey
+
+Google__SearchEngineId
+Google__ApiKey
 ```
diff --git a/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs b/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs
@@ -266,7 +266,7 @@ public async Task RagWithBingTextSearchUsingFullPagesAsync()
         // Create a kernel with OpenAI chat completion
         IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
         kernelBuilder.AddOpenAIChatCompletion(
-                modelId: TestConfiguration.OpenAI.ChatModelId,
+                modelId: TestConfiguration.OpenAI.ChatModelId, // Requires a large context window e.g. gpt-4o or gpt-4o-mini 
                 apiKey: TestConfiguration.OpenAI.ApiKey);
         Kernel kernel = kernelBuilder.Build();
 

diff --git a/...rc/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs b/...rc/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs
@@ -39,6 +39,6 @@ public class AzureCosmosDBMongoDBHotelModel(string hotelId)
     public string? Description { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }
diff --git a/...rs.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs b/...rs.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs
@@ -828,11 +828,11 @@ private sealed class VectorSearchModel
         [VectorStoreRecordData]
         public string? HotelName { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance, StoragePropertyName = "test_embedding_1")]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat, StoragePropertyName = "test_embedding_1")]
         public ReadOnlyMemory<float> TestEmbedding1 { get; set; }
 
         [BsonElement("test_embedding_2")]
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)]
         public ReadOnlyMemory<float> TestEmbedding2 { get; set; }
     }
 #pragma warning restore CA1812

diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs
@@ -39,6 +39,6 @@ public class AzureCosmosDBNoSQLHotel(string hotelId)
 
     /// <summary>A vector field.</summary>
     [JsonPropertyName("description_embedding")]
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }
diff --git a/...ectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs b/...ectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs
@@ -701,16 +701,16 @@ private sealed class TestIndexingModel
         [VectorStoreRecordKey]
         public string? Id { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 1, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 1, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
         public ReadOnlyMemory<Half>? DescriptionEmbedding1 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 2, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 2, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
         public ReadOnlyMemory<float>? DescriptionEmbedding2 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 3, IndexKind: IndexKind.QuantizedFlat, DistanceFunction: DistanceFunction.DotProductSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 3, DistanceFunction: DistanceFunction.DotProductSimilarity, IndexKind: IndexKind.QuantizedFlat)]
         public ReadOnlyMemory<byte>? DescriptionEmbedding3 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.DiskAnn, DistanceFunction: DistanceFunction.EuclideanDistance)]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.EuclideanDistance, IndexKind: IndexKind.DiskAnn)]
         public ReadOnlyMemory<sbyte>? DescriptionEmbedding4 { get; set; }
 
         [VectorStoreRecordData(IsFilterable = true)]

diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs
@@ -44,6 +44,6 @@ public sealed record WeaviateHotel
     public DateTimeOffset Timestamp { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.Hnsw)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }
diff --git a/...c/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs b/...c/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs
@@ -33,13 +33,24 @@ public VectorStoreRecordVectorAttribute(int Dimensions)
     /// Initializes a new instance of the <see cref="VectorStoreRecordVectorAttribute"/> class.
     /// </summary>
     /// <param name="Dimensions">The number of dimensions that the vector has.</param>
-    /// <param name="IndexKind">The kind of index to use.</param>
     /// <param name="DistanceFunction">The distance function to use when comparing vectors.</param>
-    public VectorStoreRecordVectorAttribute(int Dimensions, string? IndexKind, string? DistanceFunction)
+    public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction)
+    {
+        this.Dimensions = Dimensions;
+        this.DistanceFunction = DistanceFunction;
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="VectorStoreRecordVectorAttribute"/> class.
+    /// </summary>
+    /// <param name="Dimensions">The number of dimensions that the vector has.</param>
+    /// <param name="DistanceFunction">The distance function to use when comparing vectors.</param>
+    /// <param name="IndexKind">The kind of index to use.</param>
+    public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction, string? IndexKind)
     {
         this.Dimensions = Dimensions;
-        this.IndexKind = IndexKind;
         this.DistanceFunction = DistanceFunction;
+        this.IndexKind = IndexKind;
     }
 
     /// <summary>

diff --git a/...ionTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs b/...ionTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs
@@ -117,7 +117,7 @@ public record AzureCosmosDBMongoDBHotel()
         public DateTime Timestamp { get; set; }
 
         /// <summary>A vector field.</summary>
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)]
         public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
     }
 #pragma warning restore CS8618

diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs
@@ -45,6 +45,6 @@ public record AzureCosmosDBNoSQLHotel()
     public DateTimeOffset Timestamp { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs
@@ -59,6 +59,6 @@ public record PineconeAllTypes()
     [VectorStoreRecordData]
     public IEnumerable<string> Enumerable { get; set; }
 
-    [VectorStoreRecordVector(Dimensions: 8, IndexKind: null, DistanceFunction: DistanceFunction.DotProductSimilarity)]
+    [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)]
     public ReadOnlyMemory<float>? Embedding { get; set; }
 }
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs
@@ -34,6 +34,6 @@ public record PineconeHotel()
     [VectorStoreRecordData]
     public string Description { get; set; }
 
-    [VectorStoreRecordVector(Dimensions: 8, IndexKind: null, DistanceFunction: DistanceFunction.DotProductSimilarity)]
+    [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)]
     public ReadOnlyMemory<float> DescriptionEmbedding { get; set; }
 }
diff --git a/...c/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs b/...c/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs
@@ -555,7 +555,7 @@ private sealed record PineconeRecordWithUnsupportedMetric
         [VectorStoreRecordData]
         public string? Name { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 5, IndexKind: null, DistanceFunction: "just eyeball it")]
+        [VectorStoreRecordVector(Dimensions: 5, DistanceFunction: "just eyeball it")]
         public ReadOnlyMemory<float> Embedding { get; set; }
     }
 #pragma warning restore CA1812

diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs
@@ -335,7 +335,7 @@ public record HotelInfo()
         public string Description { get; set; }
 
         /// <summary>A vector field.</summary>
-        [VectorStoreRecordVector(VectorDimensions, IndexKind.Hnsw, DistanceFunction.ManhattanDistance)]
+        [VectorStoreRecordVector(VectorDimensions, DistanceFunction.ManhattanDistance, IndexKind.Hnsw)]
         public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
     }
 
@@ -358,7 +358,7 @@ public record HotelInfoWithGuidId()
         public string Description { get; set; }
 
         /// <summary>A vector field.</summary>
-        [VectorStoreRecordVector(VectorDimensions, IndexKind.Hnsw, DistanceFunction.ManhattanDistance)]
+        [VectorStoreRecordVector(VectorDimensions, DistanceFunction.ManhattanDistance, IndexKind.Hnsw)]
         public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
     }
 }

diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs
@@ -44,6 +44,6 @@ public sealed record WeaviateHotel
     public DateTimeOffset Timestamp { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.Hnsw)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }
diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs
@@ -108,8 +108,6 @@ public VectorStoreTextSearch(
         VectorStoreTextSearchOptions? options = null)
     {
         Verify.NotNull(vectorizableTextSearch);
-        Verify.NotNull(stringMapper);
-        Verify.NotNull(resultMapper);
 
         this._vectorizableTextSearch = vectorizableTextSearch;
         this._propertyReader = new Lazy<TextSearchResultPropertyReader>(() => new TextSearchResultPropertyReader(typeof(TRecord)));

diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs
@@ -552,7 +552,7 @@ private sealed class MultiPropsModel
         [JsonPropertyName("json_data2")]
         public string Data2 { get; set; } = string.Empty;
 
-        [VectorStoreRecordVector(4, IndexKind.Flat, DistanceFunction.DotProductSimilarity)]
+        [VectorStoreRecordVector(4, DistanceFunction.DotProductSimilarity, IndexKind.Flat)]
         public ReadOnlyMemory<float> Vector1 { get; set; }
 
         [VectorStoreRecordVector(StoragePropertyName = "storage_vector2")]