.Net: Added implementation of SQLite connector for new memory design

dotnet/SK-dotnet.sln

@@ -361,7 +361,19 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Process.UnitTests", "src\Ex
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GettingStartedWithProcesses", "samples\GettingStartedWithProcesses\GettingStartedWithProcesses.csproj", "{C057ACDF-DDD8-496B-BAF9-1C6E4E1248D7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Sqlite.UnitTests", "src\Connectors\Connectors.Sqlite.UnitTests\Connectors.Sqlite.UnitTests.csproj", "{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Sqlite.UnitTests", "src\Connectors\Connectors.Sqlite.UnitTests\Connectors.Sqlite.UnitTests.csproj", "{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VectorData.Abstractions", "src\Connectors\VectorData.Abstractions\VectorData.Abstractions.csproj", "{CF09AE39-67BA-4FE0-A0CD-A49BED956311}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.InMemory", "src\Connectors\Connectors.Memory.InMemory\Connectors.Memory.InMemory.csproj", "{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MemoryUnitTests", "MemoryUnitTests", "{5A7028A7-4DDF-4E4F-84A9-37CE8F8D7E89}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.InMemory.UnitTests", "src\Connectors\Connectors.InMemory.UnitTests\Connectors.InMemory.UnitTests.csproj", "{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GettingStartedWithTextSearch", "samples\GettingStartedWithTextSearch\GettingStartedWithTextSearch.csproj", "{16AFA226-E417-490D-9311-9F2099A1EEC8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VectorStoreRAG", "samples\Demos\VectorStoreRAG\VectorStoreRAG.csproj", "{28DFAF27-8FF3-4373-AAA4-2A6969C86246}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -934,6 +946,36 @@ Global
 		{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E}.Publish|Any CPU.Build.0 = Debug|Any CPU
 		{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Publish|Any CPU.Build.0 = Publish|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311}.Release|Any CPU.Build.0 = Release|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Publish|Any CPU.Build.0 = Publish|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42}.Release|Any CPU.Build.0 = Release|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{16AFA226-E417-490D-9311-9F2099A1EEC8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1062,7 +1104,13 @@ Global
 		{27AF60D6-86F5-4591-A700-4F8C93F41B11} = {0D8C6358-5DAA-4EA6-A924-C268A9A21BC9}
 		{21A32285-8443-4A75-B2E8-27E6090EC562} = {0D8C6358-5DAA-4EA6-A924-C268A9A21BC9}
 		{C057ACDF-DDD8-496B-BAF9-1C6E4E1248D7} = {FA3720F1-C99A-49B2-9577-A940257098BF}
-		{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
+		{0846F178-7EC3-4FE6-AC5C-7146E9FD9A3E} = {5A7028A7-4DDF-4E4F-84A9-37CE8F8D7E89}
+		{CF09AE39-67BA-4FE0-A0CD-A49BED956311} = {24503383-A8C4-4255-9998-28D70FE8E99A}
+		{265B8B6E-B9C4-44C7-BE96-59750F9A1DC7} = {24503383-A8C4-4255-9998-28D70FE8E99A}
+		{5A7028A7-4DDF-4E4F-84A9-37CE8F8D7E89} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
+		{A0B91C05-B3A6-436B-8F49-3427EC1CFC42} = {5A7028A7-4DDF-4E4F-84A9-37CE8F8D7E89}
+		{16AFA226-E417-490D-9311-9F2099A1EEC8} = {FA3720F1-C99A-49B2-9577-A940257098BF}
+		{28DFAF27-8FF3-4373-AAA4-2A6969C86246} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}

dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereCondition.cs

@@ -4,8 +4,14 @@
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
-internal abstract record SqliteWhereCondition(string Operand, List<object> Values, string? TableName = null)
+internal abstract class SqliteWhereCondition(string operand, List<object> values)
 {
+    public string Operand { get; set; } = operand;
+
+    public List<object> Values { get; set; } = values;
+
+    public string? TableName { get; set; }
+
     public abstract string BuildQuery(List<string> parameterNames);
 
     protected string GetOperand() => !string.IsNullOrWhiteSpace(this.TableName) ?

dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereEqualsCondition.cs

@@ -4,8 +4,8 @@
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
-internal sealed record SqliteWhereEqualsCondition(string Operand, object Value, string? TableName = null)
-    : SqliteWhereCondition(Operand, [Value], TableName)
+internal sealed class SqliteWhereEqualsCondition(string operand, object value)
+    : SqliteWhereCondition(operand, [value])
 {
     public override string BuildQuery(List<string> parameterNames)
     {

dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereInCondition.cs

@@ -4,8 +4,8 @@
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
-internal sealed record SqliteWhereInCondition(string Operand, List<object> Values, string? TableName = null)
-    : SqliteWhereCondition(Operand, Values, TableName)
+internal sealed class SqliteWhereInCondition(string operand, List<object> values)
+    : SqliteWhereCondition(operand, values)
 {
     public override string BuildQuery(List<string> parameterNames)
     {

dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereMatchCondition.cs

@@ -4,8 +4,8 @@
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
-internal sealed record SqliteWhereMatchCondition(string Operand, object Value, string? TableName = null)
-    : SqliteWhereCondition(Operand, [Value], TableName)
+internal sealed class SqliteWhereMatchCondition(string operand, object value)
+    : SqliteWhereCondition(operand, [value])
 {
     public override string BuildQuery(List<string> parameterNames)
     {

dotnet/src/Connectors/Connectors.Memory.Sqlite/ISqliteVectorStoreRecordCollectionFactory.cs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Data.Common;
-using Microsoft.SemanticKernel.Data;
+using Microsoft.Extensions.VectorData;
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
@@ -23,6 +23,5 @@ IVectorStoreRecordCollection<TKey, TRecord> CreateVectorStoreRecordCollection<TK
         DbConnection connection,
         string name,
         VectorStoreRecordDefinition? vectorStoreRecordDefinition)
-        where TKey : notnull
-        where TRecord : class;
+        where TKey : notnull;
 }

dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteConstants.cs

@@ -43,8 +43,6 @@ internal static class SqliteConstants
         typeof(decimal),
         typeof(decimal?),
         typeof(byte[]),
-        typeof(DateTime),
-        typeof(DateTime?)
     ];
 
     /// <summary>A <see cref="HashSet{T}"/> of types that vector properties on the provided model may have.</summary>

dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteGenericDataModelMapper.cs

@@ -2,7 +2,7 @@
 
 using System;
 using System.Collections.Generic;
-using Microsoft.SemanticKernel.Data;
+using Microsoft.Extensions.VectorData;
 
 namespace Microsoft.SemanticKernel.Connectors.Sqlite;
 
@@ -25,6 +25,10 @@ public SqliteGenericDataModelMapper(VectorStoreRecordPropertyReader propertyRead
         Verify.NotNull(propertyReader);
 
         this._propertyReader = propertyReader;
+
+        // Validate property types.
+        this._propertyReader.VerifyDataProperties(SqliteConstants.SupportedDataTypes, supportEnumerable: false);
+        this._propertyReader.VerifyVectorProperties(SqliteConstants.SupportedVectorTypes);
     }
 
     #region Implementation of IVectorStoreRecordMapper<VectorStoreGenericDataModel<string>, Dictionary<string, object?>>
@@ -73,7 +77,7 @@ public VectorStoreGenericDataModel<string> MapFromStorageToDataModel(Dictionary<
             {
                 if (dataModel.Data.TryGetValue(property.DataModelPropertyName, out var dataValue))
                 {
-                    properties.Add(this._propertyReader.StoragePropertyNamesMap[property.DataModelPropertyName], dataValue);
+                    properties.Add(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), dataValue);
                 }
             }
         }
@@ -90,10 +94,10 @@ public VectorStoreGenericDataModel<string> MapFromStorageToDataModel(Dictionary<
                     if (vectorValue is not null)
                     {
                         var vector = (ReadOnlyMemory<float>)vectorValue;
-                        result = SqliteVectorStoreRecordPropertyMapping.MapVector(vector);
+                        result = SqliteVectorStoreRecordPropertyMapping.MapVectorForStorageModel(vector);
                     }
 
-                    properties.Add(this._propertyReader.StoragePropertyNamesMap[property.DataModelPropertyName], result);
+                    properties.Add(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), result);
                 }
             }
         }
@@ -121,7 +125,7 @@ private VectorStoreGenericDataModel<TKey> InternalMapFromStorageToDataModel<TKey
         // Process data properties.
         foreach (var property in this._propertyReader.DataProperties)
         {
-            if (storageModel.TryGetValue(this._propertyReader.StoragePropertyNamesMap[property.DataModelPropertyName], out var dataValue))
+            if (storageModel.TryGetValue(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), out var dataValue))
             {
                 dataProperties.Add(property.DataModelPropertyName, dataValue);
             }
@@ -132,9 +136,11 @@ private VectorStoreGenericDataModel<TKey> InternalMapFromStorageToDataModel<TKey
         {
             foreach (var property in this._propertyReader.VectorProperties)
             {
-                if (storageModel.TryGetValue(this._propertyReader.StoragePropertyNamesMap[property.DataModelPropertyName], out var vectorValue))
+                if (storageModel.TryGetValue(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), out var vectorValue) &&
+                    vectorValue is byte[] vectorBytes)
                 {
-                    vectorProperties.Add(property.DataModelPropertyName, vectorValue);
+                    var vector = SqliteVectorStoreRecordPropertyMapping.MapVectorForDataModel(vectorBytes);
+                    vectorProperties.Add(property.DataModelPropertyName, vector);
                 }
             }
         }

dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteKernelBuilderExtensions.cs

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.VectorData;
 using Microsoft.SemanticKernel.Connectors.Sqlite;
-using Microsoft.SemanticKernel.Data;
 
 namespace Microsoft.SemanticKernel;
 
@@ -32,17 +32,17 @@ public static IKernelBuilder AddSqliteVectorStore(
     /// Register a SQLite <see cref="IVectorStore"/> with the specified service ID.
     /// </summary>
     /// <param name="builder">The builder to register the <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/> on.</param>
-    /// <param name="connection"><see cref="SqliteConnection"/> that will be used to manage the data in SQLite.</param>
+    /// <param name="connectionString">Connection string for <see cref="SqliteConnection"/>.</param>
     /// <param name="options">Optional options to further configure the <see cref="IVectorStore"/>.</param>
     /// <param name="serviceId">An optional service id to use as the service key.</param>
     /// <returns>Kernel builder.</returns>
     public static IKernelBuilder AddSqliteVectorStore(
         this IKernelBuilder builder,
-        SqliteConnection connection,
+        string connectionString,
         SqliteVectorStoreOptions? options = default,
         string? serviceId = default)
     {
-        builder.Services.AddSqliteVectorStore(connection, options, serviceId);
+        builder.Services.AddSqliteVectorStore(connectionString, options, serviceId);
         return builder;
     }
 
@@ -63,7 +63,6 @@ public static IKernelBuilder AddSqliteVectorStoreRecordCollection<TKey, TRecord>
         SqliteVectorStoreRecordCollectionOptions<TRecord>? options = default,
         string? serviceId = default)
         where TKey : notnull
-        where TRecord : class
     {
         builder.Services.AddSqliteVectorStoreRecordCollection<TKey, TRecord>(collectionName, options, serviceId);
         return builder;
@@ -76,20 +75,19 @@ public static IKernelBuilder AddSqliteVectorStoreRecordCollection<TKey, TRecord>
     /// <typeparam name="TRecord">The type of the record.</typeparam>
     /// <param name="builder">The builder to register the <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/> on.</param>
     /// <param name="collectionName">The name of the collection.</param>
-    /// <param name="connection"><see cref="SqliteConnection"/> that will be used to manage the data in SQLite.</param>
+    /// <param name="connectionString">Connection string for <see cref="SqliteConnection"/>.</param>
     /// <param name="options">Optional options to further configure the <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.</param>
     /// <param name="serviceId">An optional service id to use as the service key.</param>
     /// <returns>Kernel builder.</returns>
     public static IKernelBuilder AddSqliteVectorStoreRecordCollection<TKey, TRecord>(
         this IKernelBuilder builder,
         string collectionName,
-        SqliteConnection connection,
+        string connectionString,
         SqliteVectorStoreRecordCollectionOptions<TRecord>? options = default,
         string? serviceId = default)
         where TKey : notnull
-        where TRecord : class
     {
-        builder.Services.AddSqliteVectorStoreRecordCollection<TKey, TRecord>(collectionName, connection, options, serviceId);
+        builder.Services.AddSqliteVectorStoreRecordCollection<TKey, TRecord>(collectionName, connectionString, options, serviceId);
         return builder;
     }
 }