Support Ollama embedding (#67)

* Support Ollama * Update adapter version * Multiple exception handling
Zipstack · Jun 28, 2024 · f7f1427 · f7f1427
1 parent 97eaf8d
commit f7f1427
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 13 deletions.
diff --git a/pdm.lock b/pdm.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,7 +12,7 @@ dependencies = [
     "python-magic~=0.4.27",
     "python-dotenv==1.0.0",
     # LLM Triad
-    "unstract-adapters~=0.19.2",
+    "unstract-adapters~=0.20.1",
     "llama-index==0.10.38",
     "tiktoken~=0.4.0",
     "transformers==4.37.0",

diff --git a/src/unstract/sdk/__init__.py b/src/unstract/sdk/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.33.1"
+__version__ = "0.34.0"
 
 
 def get_sdk_version():

diff --git a/src/unstract/sdk/utils/callback_manager.py b/src/unstract/sdk/utils/callback_manager.py
@@ -138,7 +138,7 @@ def get_tokenizer(
                 model_name
             ).encode
             return tokenizer
-        except ValueError as e:
+        except (KeyError, ValueError) as e:
             logger.warning(str(e))
             return fallback_tokenizer