add command helpers

kreeben · Oct 19, 2022 · e3f1085 · e3f1085
1 parent ec25fe6
commit e3f1085
Show file tree

Hide file tree

Showing 15 changed files with 45 additions and 26 deletions.
diff --git a/build.bat b/build.bat
@@ -1,2 +1,6 @@
+dotnet build src/sir.httpserver --configuration Debug
+dotnet build src/sir.webfront --configuration Debug
+dotnet build src/sir.cmd --configuration Debug
 dotnet build src/sir.httpserver --configuration Release
-dotnet build src/sir --configuration Release
+dotnet build src/sir.webfront --configuration Release
+dotnet build src/sir.cmd --configuration Release
diff --git a/index.bat b/index.bat
@@ -1 +1 @@
-sir.bat indexwikipedia --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --file d:\enwiki-20211122-cirrussearch-content.json.gz --collection wikipedia --skip 0 --take 500 --pageSize 100 --sampleSize 100
+sir.bat indexwikipedia --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --file d:\enwiki-20211122-cirrussearch-content.json.gz --collection wikipedia --skip 0 --take 10000 --pageSize 1000 --sampleSize 100
diff --git a/src/Sir.Core/Field.cs b/src/Sir.Core/Field.cs
@@ -44,7 +44,7 @@ public void Analyze<T>(IModel<T> model, bool label, IStreamDispatcher streamDisp
             {
                 foreach (var token in tokens)
                 {
-                    model.ExecutePut<string>(Tree, new VectorNode(token, keyId: KeyId), reader);
+                    model.Put<string>(Tree, new VectorNode(token, keyId: KeyId), reader);
                 }
 
                 _tokens = GetTokens();

diff --git a/src/Sir.Core/IIndexReadWriteStrategy.cs b/src/Sir.Core/IIndexReadWriteStrategy.cs
@@ -2,7 +2,7 @@
 {
     public interface IIndexReadWriteStrategy
     {
-        void ExecutePut<T>(VectorNode column, VectorNode node, IColumnReader reader);
-        Hit ExecuteGetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader);
+        void Put<T>(VectorNode column, VectorNode node, IColumnReader reader);
+        Hit GetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader);
     }
 }
diff --git a/src/Sir.Core/IO/GraphBuilder.cs b/src/Sir.Core/IO/GraphBuilder.cs
@@ -14,7 +14,7 @@ public static VectorNode CreateTree<T>(this IModel<T> model, IIndexReadWriteStra
             {
                 foreach (var vector in model.CreateEmbedding(item, true))
                 {
-                    indexingStrategy.ExecutePut<T>(root, new VectorNode(vector), reader);
+                    indexingStrategy.Put<T>(root, new VectorNode(vector), reader);
                 }
             }
 

diff --git a/src/Sir.Core/IndexingStrategies/OptimizedPageIndexingStrategy.cs b/src/Sir.Core/IndexingStrategies/OptimizedPageIndexingStrategy.cs
@@ -11,14 +11,14 @@ public OptimizedPageIndexingStrategy(IModel model)
             _model = model;
         }
 
-        public Hit ExecuteGetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
+        public Hit GetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
         {
             return reader.ClosestMatchOrNullStoppingAtBestPage(vector, model);
         }
 
-        public void ExecutePut<T>(VectorNode column, VectorNode node, IColumnReader reader)
+        public void Put<T>(VectorNode column, VectorNode node, IColumnReader reader)
         {
-            var existing = reader.ClosestMatchOrNullScanningAllPages(node.Vector, _model);
+            var existing = reader.ClosestMatchOrNullStoppingAtBestPage(node.Vector, _model);
 
             if (existing != null && existing.Score >= _model.IdenticalAngle)
             {

diff --git a/src/Sir.Images/LinearClassifierImageModel.cs b/src/Sir.Images/LinearClassifierImageModel.cs
@@ -11,7 +11,7 @@ public class LinearClassifierImageModel : DistanceCalculator, IModel<IImage>
         public double FoldAngle => 0.75d;
         public override int NumOfDimensions => 784; 
 
-        public void ExecutePut<T>(VectorNode column, VectorNode node, IColumnReader reader)
+        public void Put<T>(VectorNode column, VectorNode node, IColumnReader reader)
         {
             column.AddOrAppendSupervised(node, this);
         }
@@ -23,7 +23,7 @@ public IEnumerable<ISerializableVector> CreateEmbedding(IImage data, bool label)
             yield return new SerializableVector(pixels, data.Label);
         }
 
-        public Hit ExecuteGetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
+        public Hit GetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
         {
             return reader.ClosestMatchOrNullScanningAllPages(vector, model);
         }

diff --git a/src/Sir.Search/InMemoryIndexSession.cs b/src/Sir.Search/InMemoryIndexSession.cs
@@ -61,7 +61,7 @@ public void Put(VectorNode documentTree)
 
             foreach (var node in PathFinder.All(documentTree))
             {
-                _indexingStrategy.ExecutePut<T>(
+                _indexingStrategy.Put<T>(
                     column, 
                     new VectorNode(node.Vector, docIds: node.DocIds), 
                     GetReader(documentTree.KeyId.Value));

diff --git a/src/Sir.Search/SearchSession.cs b/src/Sir.Search/SearchSession.cs
@@ -120,7 +120,7 @@ private void Scan(IQuery query)
 
                     if (reader != null)
                     {
-                        var hit =_model.ExecuteGetClosestMatchOrNull(term.Vector, _model, reader);
+                        var hit =_model.GetClosestMatchOrNull(term.Vector, _model, reader);
 
                         if (hit != null)
                         {

diff --git a/src/Sir.Search/SessionFactory.cs b/src/Sir.Search/SessionFactory.cs
@@ -89,17 +89,25 @@ public long GetDocCount(string directory, string collection)
         public void Truncate(string directory, ulong collectionId)
         {
             var count = 0;
-            var key = Path.Combine(directory, collectionId.ToString()).ToHash();
 
-            foreach (var file in Directory.GetFiles(directory, $"{collectionId}*"))
+            if (Directory.Exists(directory))
             {
-                File.Delete(file);
-                count++;
-            }
+                var path = Path.Combine(directory, collectionId.ToString());
+                var key = path.ToHash();
 
-            lock (_syncKeys)
-            {
-                _keys.Remove(key, out _);
+                if (Directory.Exists(path))
+                {
+                    foreach (var file in Directory.GetFiles(path, $"{collectionId}*"))
+                    {
+                        File.Delete(file);
+                        count++;
+                    }
+
+                    lock (_syncKeys)
+                    {
+                        _keys.Remove(key, out _);
+                    }
+                }
             }
 
             LogInformation($"truncated collection {collectionId} ({count} files affected)");

diff --git a/src/Sir.Strings/BagOfCharsModel.cs b/src/Sir.Strings/BagOfCharsModel.cs
@@ -9,7 +9,7 @@ public class BagOfCharsModel : DistanceCalculator, IModel<string>
         public double FoldAngle => 0.55d;
         public override int NumOfDimensions => System.Text.Unicode.UnicodeRanges.All.Length;
 
-        public void ExecutePut<T>(VectorNode column, VectorNode node, IColumnReader reader)
+        public void Put<T>(VectorNode column, VectorNode node, IColumnReader reader)
         {
             column.AddOrAppend(node, this);
         }
@@ -65,7 +65,7 @@ public IEnumerable<ISerializableVector> CreateEmbedding(string data, bool label)
             }
         }
 
-        public Hit ExecuteGetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
+        public Hit GetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
         {
             return reader.ClosestMatchOrNullScanningAllPages(vector, model);
         }

diff --git a/src/Sir.Strings/NGramModel.cs b/src/Sir.Strings/NGramModel.cs
@@ -17,7 +17,7 @@ public NGramModel(BagOfCharsModel wordTokenizer)
             NumOfDimensions = wordTokenizer.NumOfDimensions * 2;
         }
 
-        public void ExecutePut<T>(VectorNode column, VectorNode node, IColumnReader reader)
+        public void Put<T>(VectorNode column, VectorNode node, IColumnReader reader)
         {
             column.AddOrAppend(node, this);
         }
@@ -51,7 +51,7 @@ public IEnumerable<ISerializableVector> CreateEmbedding(string data, bool label)
             }
         }
 
-        public Hit ExecuteGetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
+        public Hit GetClosestMatchOrNull(ISerializableVector vector, IModel model, IColumnReader reader)
         {
             return reader.ClosestMatchOrNullScanningAllPages(vector, model);
         }

diff --git a/src/Sir.sln b/src/Sir.sln
@@ -10,11 +10,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{
 		AssemblyInfo.cs = AssemblyInfo.cs
 		..\build.bat = ..\build.bat
 		..\compare.bat = ..\compare.bat
+		..\index.bat = ..\index.bat
 		..\INSTALL.md = ..\INSTALL.md
 		..\push-to-nuget.bat = ..\push-to-nuget.bat
 		..\README.md = ..\README.md
 		..\sir.bat = ..\sir.bat
+		..\test.bat = ..\test.bat
+		..\truncate.bat = ..\truncate.bat
 		..\USER-GUIDE.md = ..\USER-GUIDE.md
+		..\validate.bat = ..\validate.bat
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sir.HttpServer", "Sir.HttpServer\Sir.HttpServer.csproj", "{C94C2F5D-AE55-4157-A74A-26D49EE73E96}"

diff --git a/test.bat b/test.bat
@@ -0,0 +1,3 @@
+call sir.bat truncate --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --collection wikipedia
+call index.bat
+call validate.bat
diff --git a/validate.bat b/validate.bat
@@ -1 +1 @@
-sir.bat validate --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --collection wikipedia --skip 0 --take 1000
+sir.bat validate --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --collection wikipedia --skip 0 --take 10000
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		sir.bat indexwikipedia --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --file d:\enwiki-20211122-cirrussearch-content.json.gz --collection wikipedia --skip 0 --take 500 --pageSize 100 --sampleSize 100
		sir.bat indexwikipedia --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --file d:\enwiki-20211122-cirrussearch-content.json.gz --collection wikipedia --skip 0 --take 10000 --pageSize 1000 --sampleSize 100
-Original file line number
+Diff line change
@@ Expand Up @@
                 {
                     foreach (var vector in model.CreateEmbedding(item, true))
                     {
-                        indexingStrategy.ExecutePut<T>(root, new VectorNode(vector), reader);
+                        indexingStrategy.Put<T>(root, new VectorNode(vector), reader);
                     }
                 }
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		sir.bat validate --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --collection wikipedia --skip 0 --take 1000
		sir.bat validate --directory C:\projects\resin\src\Sir.HttpServer\AppData\database --collection wikipedia --skip 0 --take 10000