implement paged postings writer

src/AssemblyInfo.cs

@@ -1,4 +1,4 @@
 using System.Reflection;
 
-[assembly: AssemblyVersion("0.4.0.7")]
-[assembly: AssemblyFileVersion("0.4.0.7")]
+[assembly: AssemblyVersion("0.5.0.2")]
+[assembly: AssemblyFileVersion("0.5.0.2")]

src/Sir.Cmd/Sir.Cmd.csproj

@@ -12,10 +12,12 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Sir.CommonCrawl\Sir.CommonCrawl.csproj" />
+    <ProjectReference Include="..\Sir.CommonCrawl\Sir.DataProvider.CommonCrawl.csproj" />
+    <ProjectReference Include="..\Sir.Images\Sir.Model.Images.csproj" />
     <ProjectReference Include="..\Sir.InformationRetreival\Sir.InformationRetrieval.csproj" />
-    <ProjectReference Include="..\Sir.Mnist\Sir.Mnist.csproj" />
-    <ProjectReference Include="..\Sir.Wikipedia\Sir.Wikipedia.csproj" />
+    <ProjectReference Include="..\Sir.Mnist\Sir.DataProvider.Mnist.csproj" />
+    <ProjectReference Include="..\Sir.Strings\Sir.Model.Strings.csproj" />
+    <ProjectReference Include="..\Sir.Wikipedia\Sir.DataProvider.Wikipedia.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/Sir.Mnist/Sir.Mnist.csproj → src/Sir.CommonCrawl/Sir.DataProvider.CommonCrawl.csproj

@@ -13,8 +13,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Sir.Images\Sir.Images.csproj" />
     <ProjectReference Include="..\Sir.InformationRetreival\Sir.InformationRetrieval.csproj" />
+    <ProjectReference Include="..\Sir.Strings\Sir.Model.Strings.csproj" />
   </ItemGroup>
 
 </Project>

src/Sir.Document/Field.cs

@@ -7,15 +7,13 @@ namespace Sir.Documents
     [DebuggerDisplay("{Name}")]
     public class Field
     {
-        private IEnumerable<ISerializableVector> _tokens;
-
         public long KeyId { get; set; }
         public long DocumentId { get; set; }
         public string Name { get; }
         public object Value { get; set; }
-        public IEnumerable<ISerializableVector> Tokens { get { return _tokens; } }
+        public IEnumerable<ISerializableVector> Tokens { get; }
 
-        public Field(string name, object value, long keyId = -1, long documentId = -1)
+        public Field(string name, object value, long keyId = -1, long documentId = -1, IEnumerable<ISerializableVector> tokens = null)
         {
             if (name is null) throw new ArgumentNullException(nameof(name));
             if (value == null) throw new ArgumentNullException(nameof(value));
@@ -24,6 +22,7 @@ public Field(string name, object value, long keyId = -1, long documentId = -1)
             Value = value;
             KeyId = keyId;
             DocumentId = documentId;
+            Tokens = tokens;
         }
     }
 }

src/Sir.HttpServer/HttpQueryParser.cs

@@ -20,7 +20,7 @@ public HttpQueryParser(QueryParser<string> parser)
             _parser = parser;
         }
 
-        public async Task<IQuery> ParseRequest(HttpRequest request, IEnumerable<string> collections = null)
+        public async Task<Query> ParseRequest(HttpRequest request, IEnumerable<string> collections = null)
         {
             var select = request.Query["select"].ToArray();
 
@@ -56,20 +56,20 @@ public static async Task<dynamic> DeserializeFromStream(Stream stream)
             }
         }
 
-        public IQuery ParseFormattedString(string formattedQuery, string[] select)
+        public Query ParseFormattedString(string formattedQuery, string[] select)
         {
             var document = JsonConvert.DeserializeObject<IDictionary<string, object>>(
                 formattedQuery, new JsonConverter[] { new DictionaryConverter() });
 
             return ParseDictionary(document, select);
         }
 
-        public IQuery ParseDictionary(IDictionary<string, object> document, string[] select)
+        public Query ParseDictionary(IDictionary<string, object> document, string[] select)
         {
             return _parser.Parse(document, select, true);
         }
 
-        private void DoParseQuery(IQuery query, IDictionary<string, object> result)
+        private void DoParseQuery(Query query, IDictionary<string, object> result)
         {
             if (result == null)
                 return;
@@ -114,7 +114,7 @@ private void DoParseQuery(IQuery query, IDictionary<string, object> result)
             }
         }
 
-        public void ParseQuery(IQuery query, IDictionary<string, object> result)
+        public void ParseQuery(Query query, IDictionary<string, object> result)
         {
             DoParseQuery(query, result);
         }

src/Sir.HttpServer/Sir.HttpServer.csproj

@@ -46,7 +46,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Sir.InformationRetreival\Sir.InformationRetrieval.csproj" />
-    <ProjectReference Include="..\Sir.Strings\Sir.Strings.csproj" />
+    <ProjectReference Include="..\Sir.Strings\Sir.Model.Strings.csproj" />
   </ItemGroup>
 
 </Project>

src/Sir.ImageTests/Sir.ImageTests.csproj

@@ -15,10 +15,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Sir.Core\Sir.InformationRetreival.csproj" />
-    <ProjectReference Include="..\Sir.Images\Sir.Images.csproj" />
+    <ProjectReference Include="..\Sir.Images\Sir.Model.Images.csproj" />
     <ProjectReference Include="..\Sir.InformationRetreival\Sir.InformationRetrieval.csproj" />
-    <ProjectReference Include="..\Sir.Mnist\Sir.Mnist.csproj" />
+    <ProjectReference Include="..\Sir.Mnist\Sir.DataProvider.Mnist.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/Sir.InformationRetreival/Hit.cs

@@ -11,9 +11,17 @@ public class Hit
         public List<long> PostingsOffsets { get; set; }
 
         public Hit (VectorNode node, double score)
+        {
+            Node = node ?? throw new System.ArgumentNullException(nameof(node));
+            Score = score;
+            PostingsOffsets = new List<long> { node.PostingsOffset };
+        }
+
+        public Hit(VectorNode node, double score, long postingsOffset)
         {
             Score = score;
             Node = node;
+            PostingsOffsets = new List<long> { postingsOffset };
         }
 
         public override string ToString()

src/Sir.InformationRetreival/IIndexReadWriteStrategy.cs

@@ -7,6 +7,6 @@ public interface IIndexReadWriteStrategy
     {
         void Put<T>(VectorNode column, VectorNode node);
         Hit GetMatchOrNull(ISerializableVector vector, IModel model, ColumnReader reader);
-        void Commit(string directory, ulong collectionId, long keyId, VectorNode tree, ILogger logger = null);
+        void SerializePage(string directory, ulong collectionId, long keyId, VectorNode tree, IndexCache indexCache, ILogger logger = null);
     }
 }

src/Sir.InformationRetreival/IO/ColumnReader.cs

@@ -1,5 +1,4 @@
-using Microsoft.Extensions.Logging;
-using System;
+using System;
 using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
@@ -36,7 +35,7 @@ public Hit ClosestMatchOrNullScanningAllPages(ISerializableVector vector, IModel
             {
                 var hit = ClosestMatchInPage(vector, model, page.offset);
 
-                if (hit.Score > 0)
+                if (hit != null && hit.Score > 0)
                 {
                     hits.Add(hit);
                 }
@@ -71,7 +70,7 @@ public Hit ClosestMatchOrNullStoppingAtFirstIdenticalPage(ISerializableVector ve
             {
                 var hit = ClosestMatchInPage(vector, model, page.offset);
 
-                if (hit.Score > 0)
+                if (hit != null && hit.Score > 0)
                 {
                     if (best == null || hit.Score > best.Score)
                     {
@@ -82,10 +81,10 @@ public Hit ClosestMatchOrNullStoppingAtFirstIdenticalPage(ISerializableVector ve
                     {
                         best.PostingsOffsets.Add(hit.Node.PostingsOffset);
                     }
-                }
 
-                if (hit.Score.Approximates(model.IdenticalAngle))
-                    break;
+                    if (hit.Score.Approximates(model.IdenticalAngle))
+                        break;
+                }
             }
 
             return best;
@@ -110,7 +109,7 @@ private Hit ClosestMatchInPage(ISerializableVector queryVector, IModel model, lo
 
                 var angle = model.CosAngle(queryVector, vecOffset, (int)componentCount, _vectorFile);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle >= model.IdenticalAngle || angle.Approximates(model.IdenticalAngle))
                 {
                     bestScore = angle;
                     bestNode = new VectorNode(postingsOffset: postingsOffset);
@@ -124,7 +123,7 @@ private Hit ClosestMatchInPage(ISerializableVector queryVector, IModel model, lo
                         bestScore = angle;
                         bestNode = new VectorNode(postingsOffset: postingsOffset);
                     }
-                    else if (angle == bestScore)
+                    else if (angle.Approximates(bestScore))
                     {
                         bestNode.PostingsOffset = postingsOffset;
                     }
@@ -153,7 +152,7 @@ private Hit ClosestMatchInPage(ISerializableVector queryVector, IModel model, lo
                         bestScore = angle;
                         bestNode = new VectorNode(postingsOffset: postingsOffset);
                     }
-                    else if (angle > 0 && angle == bestScore)
+                    else if (angle > 0 && angle.Approximates(bestScore))
                     {
                         bestNode.PostingsOffset = postingsOffset;
                     }
@@ -188,7 +187,7 @@ private Hit ClosestMatchInPage(ISerializableVector queryVector, IModel model, lo
 
             ArrayPool<byte>.Shared.Return(block);
 
-            return new Hit(bestNode, bestScore);
+            return bestNode == null ? null : new Hit(bestNode, bestScore);
         }
 
         private void SkipTree()

src/Sir.InformationRetreival/IO/GraphBuilder.cs

@@ -33,7 +33,7 @@ public static void AddOrAppendSupervised(
             {
                 var angle = cursor.Vector == null ? 0 : model.CosAngle(node.Vector, cursor.Vector);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle >= model.IdenticalAngle || angle.Approximates(model.IdenticalAngle))
                 {
                     if (!cursor.Vector.Label.Equals(node.Vector.Label))
                         throw new InvalidOperationException($"IdenticalAngle {model.IdenticalAngle} is too low. Angle was {angle}");
@@ -79,7 +79,7 @@ public static void AddOrAppend(
             {
                 var angle = cursor.Vector == null ? 0 : model.CosAngle(node.Vector, cursor.Vector);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle.Approximates(model.IdenticalAngle))
                 {
                     AppendDocIds(cursor, node);
 
@@ -123,7 +123,7 @@ public static void AddIfUnique(
             {
                 var angle = cursor.Vector == null ? 0 : model.CosAngle(node.Vector, cursor.Vector);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle.Approximates(model.IdenticalAngle))
                 {
                     break;
                 }
@@ -165,7 +165,7 @@ public static bool TryAdd(
             {
                 var angle = cursor.Vector == null ? 0 : model.CosAngle(node.Vector, cursor.Vector);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle.Approximates(model.IdenticalAngle))
                 {
                     return false;
                 }
@@ -209,7 +209,7 @@ public static void Build(
             {
                 var angle = cursor.Vector == null ? 0 : model.CosAngle(node.Vector, cursor.Vector);
 
-                if (angle >= model.IdenticalAngle)
+                if (angle.Approximates(model.IdenticalAngle))
                 {
                     break;
                 }
@@ -240,8 +240,11 @@ public static void Build(
             }
         }
 
-        public static void AppendDocIds(this VectorNode target, VectorNode source)
+        private static void AppendDocIds(this VectorNode target, VectorNode source)
         {
+            if (target.DocIds == null || source.DocIds == null)
+                return;
+
             foreach (var d in source.DocIds)
                 target.DocIds.Add(d);
         }
@@ -297,7 +300,7 @@ public static (long offset, long length) SerializeTree(this VectorNode node, Str
             {
                 if (node.PostingsOffset == -1 && postingsWriter != null)
                 {
-                    postingsWriter.SerializePostings(node);
+                    node.PostingsOffset = postingsWriter.SerializePostings(node);
                 }
 
                 if (vectorStream != null)

src/Sir.InformationRetreival/IO/PathFinder.cs

@@ -25,7 +25,7 @@ public static Hit ClosestMatch(VectorNode root, ISerializableVector vector, IMod
                         best = cursor;
                     }
 
-                    if (angle >= model.IdenticalAngle)
+                    if (angle >= model.IdenticalAngle || angle.Approximates(model.IdenticalAngle))
                     {
                         break;
                     }

src/Sir.InformationRetreival/IO/PostingsReader.cs

@@ -25,10 +25,10 @@ public PostingsReader(string directory, ulong collectionId, long keyId, ILogger
             _collectionId = collectionId;
         }
 
-        public IList<(ulong, long)> Read(long keyId, IList<long> offsets)
+        public HashSet<(ulong, long)> Read(long keyId, IList<long> offsets)
         {
             var time = Stopwatch.StartNew();
-            var documents = new List<(ulong, long)>();
+            var documents = new HashSet<(ulong, long)>(); // collection ID, document ID
 
             foreach (var offset in offsets)
                 GetPostingsFromStream(keyId, offset, documents);
@@ -39,20 +39,21 @@ public PostingsReader(string directory, ulong collectionId, long keyId, ILogger
             return documents;
         }
 
-        private void GetPostingsFromStream(long keyId, long postingsOffset, List<(ulong collectionId, long docId)> documents)
+        private void GetPostingsFromStream(long keyId, long postingsOffset, HashSet<(ulong collectionId, long docId)> postings)
         {
+            // seek to page
             _stream.Seek(postingsOffset, SeekOrigin.Begin);
 
             var headerLen = sizeof(long) * 2;
-            var headerBuf = ArrayPool<byte>.Shared.Rent(headerLen);
 
+            // read header
+            var headerBuf = ArrayPool<byte>.Shared.Rent(headerLen);
             _stream.Read(headerBuf, 0, headerLen);
-
             var numOfPostings = BitConverter.ToInt64(headerBuf);
             var addressOfNextPage = BitConverter.ToInt64(headerBuf, sizeof(long));
-
             ArrayPool<byte>.Shared.Return(headerBuf);
 
+            // read postings
             var listLen = sizeof(long) * numOfPostings;
             var listBuf = new byte[listLen];
             var read = _stream.Read(listBuf);
@@ -62,12 +63,12 @@ private void GetPostingsFromStream(long keyId, long postingsOffset, List<(ulong
 
             foreach (var docId in MemoryMarshal.Cast<byte, long>(listBuf))
             {
-                documents.Add((_collectionId, docId));
+                postings.Add((_collectionId, docId));
             }
 
             if (addressOfNextPage > 0)
             {
-                GetPostingsFromStream(keyId, addressOfNextPage, documents);
+                GetPostingsFromStream(keyId, addressOfNextPage, postings);
             }
         }