Model File Manager

LLama.Unittest/Constants.cs

@@ -4,6 +4,7 @@ namespace LLama.Unittest
 {
     internal static class Constants
     {
+        public static readonly string ModelDirectory = "Models";
         public static readonly string GenerativeModelPath = "Models/llama-2-7b-chat.Q3_K_S.gguf";
         public static readonly string EmbeddingModelPath = "Models/all-MiniLM-L12-v2.Q8_0.gguf";
 

LLama.Unittest/LLama.Unittest.csproj

@@ -32,8 +32,6 @@
     <DownloadFile SourceUrl="https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/llava-v1.6-mistral-7b.Q3_K_XS.gguf" DestinationFolder="Models" DestinationFileName="llava-v1.6-mistral-7b.Q3_K_XS.gguf" SkipUnchangedFiles="true"></DownloadFile>
     <DownloadFile SourceUrl="https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/mmproj-model-f16.gguf" DestinationFolder="Models" DestinationFileName="mmproj-model-f16.gguf" SkipUnchangedFiles="true"></DownloadFile>
     <DownloadFile SourceUrl="https://huggingface.co/leliuga/all-MiniLM-L12-v2-GGUF/resolve/main/all-MiniLM-L12-v2.Q8_0.gguf" DestinationFolder="Models" DestinationFileName="all-MiniLM-L12-v2.Q8_0.gguf" SkipUnchangedFiles="true"></DownloadFile>
-
-
   </Target>
 
   <ItemGroup>

LLama.Unittest/Model/FileSystemModelRepoTests.cs

@@ -0,0 +1,104 @@
+using LLama.Model;
+
+namespace LLama.Unittest.Model;
+
+public class FileSystemModelRepoTests
+{
+    private readonly FileSystemModelRepo TestableRepo;
+
+    public FileSystemModelRepoTests()
+    {
+        TestableRepo = new([Constants.ModelDirectory]);
+    }
+
+    [Fact]
+    public void ModelDirectories_IsCorrect()
+    {
+        var dirs = TestableRepo.ListSources();
+        Assert.Single(dirs);
+
+        var expected = dirs.First()!.Contains(Constants.ModelDirectory);
+        Assert.True(expected);
+    }
+
+    [Fact]
+    public void AddDirectory_DoesntDuplicate()
+    {
+        for (var i = 0; i < 10; i++)
+        {
+            TestableRepo.AddSource(Constants.ModelDirectory);
+            TestableRepo.AddSource(Path.GetFullPath(Constants.ModelDirectory));
+
+            var dirs = TestableRepo.ListSources();
+            Assert.Single(dirs);
+            var expected = dirs.First()!.Contains(Constants.ModelDirectory);
+            Assert.True(expected);
+        }
+    }
+
+    [Fact]
+    public void RemoveDirectory()
+    {
+        var dirs = TestableRepo.ListSources();
+        Assert.Single(dirs);
+        var expected = dirs.First()!.Contains(Constants.ModelDirectory);
+        Assert.True(expected);
+
+        Assert.True(TestableRepo.RemoveSource(Constants.ModelDirectory));
+        Assert.Empty(TestableRepo.ListSources());
+        Assert.Empty(TestableRepo.GetAvailableModels());
+    }
+
+    [Fact]
+    public void RemoveDirectory_DoesNotExist()
+    {
+        var dirs = TestableRepo.ListSources();
+        Assert.Single(dirs);
+        var expected = dirs.First()!.Contains(Constants.ModelDirectory);
+        Assert.True(expected);
+
+        Assert.False(TestableRepo.RemoveSource("foo/boo/bar"));
+        Assert.Single(dirs);
+    }
+
+    [Fact]
+    public void RemoveAllDirectories()
+    {
+        var dirs = TestableRepo.ListSources();
+        Assert.Single(dirs);
+        var expected = dirs.First()!.Contains(Constants.ModelDirectory);
+        Assert.True(expected);
+
+        TestableRepo.RemoveAllSources();
+        Assert.Empty(TestableRepo.ListSources());
+        Assert.Empty(TestableRepo.GetAvailableModels());
+    }
+
+    [Fact]
+    public void ModelFiles_IsCorrect()
+    {
+        var files = TestableRepo.GetAvailableModels();
+        Assert.Equal(4, files.Count());
+    }
+
+    [Fact]
+    public void GetAvailableModelsFromDirectory()
+    {
+        var files = TestableRepo.GetAvailableModelsFromSource(Constants.ModelDirectory);
+        Assert.Equal(4, files.Count());
+
+        files = TestableRepo.GetAvailableModels();
+        Assert.Equal(4, files.Count());
+    }
+
+    [Fact]
+    public void TryGetModelFileMetadata_WhenExists()
+    {
+        var expectedFile = TestableRepo.GetAvailableModels().First();
+        var found = TestableRepo.TryGetModelFileMetadata(expectedFile.ModelFileUri, out var foundData);
+
+        Assert.True(found);
+        Assert.Equal(expectedFile.ModelFileUri, foundData.ModelFileUri);
+    }
+
+}

LLama.Unittest/Model/ModelCacheTests.cs

@@ -0,0 +1,125 @@
+using LLama.Common;
+using LLama.Model;
+
+namespace LLama.Unittest.Model;
+
+public class ModelManagerTests
+{
+    private readonly IModelSourceRepo _testRepo = new FileSystemModelRepo([Constants.ModelDirectory]);
+
+    private readonly ModelCache TestableModelManager;
+
+    public ModelManagerTests()
+    {
+        TestableModelManager = new();
+    }
+
+    [Fact]
+    public async void LoadModel_DisposesOnUnload()
+    {
+        const string modelId = "llama-2-7b";
+        var modelToLoad = _testRepo.GetAvailableModels()
+            .First(f => f.ModelFileName.Contains(modelId));
+
+        // Load success
+        var model = await TestableModelManager.LoadModelAsync(modelToLoad, modelId);
+        Assert.NotNull(model);
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // Load with same Id throws
+        await Assert.ThrowsAsync<ArgumentException>(async () =>
+        {
+            await TestableModelManager.LoadModelAsync(modelToLoad, modelId);
+            return;
+        });
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // unloaded and disposed
+        Assert.True(TestableModelManager.UnloadModel(modelId));
+        Assert.Throws<ObjectDisposedException>(() =>
+        {
+            _ = model.CreateContext(new ModelParams(modelToLoad.ModelFileUri));
+        });
+        Assert.Equal(0, TestableModelManager.ModelsCached());
+
+        // already unloaded and disposed
+        Assert.False(TestableModelManager.UnloadModel(modelId));
+        Assert.Throws<ObjectDisposedException>(() =>
+        {
+            _ = model.CreateContext(new ModelParams(modelToLoad.ModelFileUri));
+        });
+
+        // Can be reloaded after unload
+        model = await TestableModelManager.LoadModelAsync(modelToLoad, modelId);
+        Assert.NotNull(model);
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+        Assert.True(TestableModelManager.UnloadModel(modelId));
+        Assert.Equal(0, TestableModelManager.ModelsCached());
+    }
+
+    [Fact]
+    public async void TryCloneLoadedModel_ClonesAndCaches()
+    {
+        const string modelId = "llama-2-7b";
+        var modelToLoad = _testRepo.GetAvailableModels()
+            .First(f => f.ModelFileName.Contains(modelId));
+
+        var model = await TestableModelManager.LoadModelAsync(modelToLoad, modelId);
+        Assert.NotNull(model);
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // clone it -- Ref 2
+        const string cloneId = nameof(cloneId);
+        var isCachedAndCloned = TestableModelManager.TryCloneLoadedModel(modelId, cloneId, out var cachedModel);
+        Assert.True(isCachedAndCloned);
+        Assert.NotNull(cachedModel);
+        Assert.Equal(2, TestableModelManager.ModelsCached());
+
+        cachedModel.Dispose(); //-- ref 1
+        Assert.True(TestableModelManager.UnloadModel(modelId));
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // unloaded and disposed` -- ref 2
+        Assert.True(TestableModelManager.UnloadModel(cloneId));
+        Assert.Equal(0, TestableModelManager.ModelsCached());
+
+        Assert.False(TestableModelManager.UnloadModel(modelId));
+        Assert.False(TestableModelManager.UnloadModel(cloneId));
+        Assert.Throws<ObjectDisposedException>(() =>
+        {
+            _ = model.CreateContext(new ModelParams(modelToLoad.ModelFileUri));
+        });
+        Assert.Throws<ObjectDisposedException>(() =>
+        {
+            _ = cachedModel.CreateContext(new ModelParams(modelToLoad.ModelFileUri));
+        });
+    }
+
+    [Fact]
+    public async void TryCloneLoadedModel_SameId_Throws()
+    {
+        const string modelId = "llama-2-7b";
+        var modelToLoad = _testRepo.GetAvailableModels()
+            .First(f => f.ModelFileName.Contains(modelId));
+
+        var model = await TestableModelManager.LoadModelAsync(modelToLoad, modelId);
+        Assert.NotNull(model);
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // Same Id clone fails
+        Assert.Throws<ArgumentException>(() =>
+        {
+            TestableModelManager.TryCloneLoadedModel(modelId, modelId, out var cachedModel);
+        });
+        Assert.Equal(1, TestableModelManager.ModelsCached());
+
+        // Unload and dispose
+        Assert.True(TestableModelManager.UnloadModel(modelId));
+        Assert.Equal(0, TestableModelManager.ModelsCached());
+        Assert.False(TestableModelManager.UnloadModel(modelId));
+        Assert.Throws<ObjectDisposedException>(() =>
+        {
+            _ = model.CreateContext(new ModelParams(modelToLoad.ModelFileUri));
+        });
+    }
+}