Add support for using unquantized models in the GGUF format from the …

…source
countzero · Feb 22, 2024 · 1fd029e · 1fd029e
1 parent 49edaf6
commit 1fd029e
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.0] - 2024-02-22
+
+### Added
+- Add support for using unquantized models in the GGUF format from the source
+
 ## [1.2.0] - 2024-02-20
 
 ### Added

diff --git a/quantize_weights_for_llama.cpp.ps1 b/quantize_weights_for_llama.cpp.ps1
@@ -40,9 +40,17 @@ ForEach ($repositoryName in $repositoryDirectories) {
 
     Write-Host "Working on ${repositoryName}..." -ForegroundColor "DarkYellow"
 
-    $unquantizedModelPath = Join-Path -Path $cacheDirectory -ChildPath "${repositoryName}.unquantized.gguf"
+    $unquantizedModelPath = Join-Path -Path $cacheDirectory -ChildPath "${repositoryName}.gguf"
     $importanceMatrixPath = Join-Path -Path $cacheDirectory -ChildPath "${repositoryName}.importance-matrix.dat"
 
+    # If a repository already contains an unquantized GGUF file we are using it directly.
+    $unquantizedModelPathFromSource = Join-Path -Path $sourceDirectory -ChildPath $repositoryName | Join-Path -ChildPath "${repositoryName}.gguf"
+    $unqantizedModelAvailableInSource = (Test-Path -Path $unquantizedModelPathFromSource)
+    if ($unqantizedModelAvailableInSource) {
+        Write-Host "Found unquantized model $unquantizedModelPathFromSource in source, skipping conversion..." -ForegroundColor "DarkYellow"
+        $unquantizedModelPath = $unquantizedModelPathFromSource
+    }
+
     ForEach ($type in $quantizationTypes) {
 
         $quantizedModelPath = Join-Path -Path $targetDirectoryPath -ChildPath "${repositoryName}.${type}.gguf"
@@ -100,13 +108,16 @@ ForEach ($repositoryName in $repositoryDirectories) {
         }
     }
 
-    # Note that we are not removing *.importance-matrix.dat files because
-    # they are relatively small but take a _very_ long time to compute.
-    if (Test-Path -Path $unquantizedModelPath) {
+    # We are exclusively removing unqantized models we created.
+    # An unquantized model in the repository is left untouched.
+    if ((Test-Path -Path $unquantizedModelPath) -and !($unqantizedModelAvailableInSource)) {
 
         Write-Host "Removing intermediate unquantized model ${unquantizedModelPath}..." -ForegroundColor "DarkYellow"
         Remove-Item "${unquantizedModelPath}" -Recurse -Force
     }
+
+    # Note that we are not removing *.importance-matrix.dat files because
+    # they are relatively small but take a _very_ long time to compute.
 }
 
 $stopwatch.Stop()