Index and LLM fixes (#36)

* Bypass x2text for text file

* Regex fix for JSON

* Version bump

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
     "python-magic~=0.4.27",
     "python-dotenv==1.0.0",
     # LLM Triad
-    "unstract-adapters~=0.9.0",
+    "unstract-adapters~=0.10.0",
     "llama-index==0.10.28",
     "tiktoken~=0.4.0",
     "transformers==4.37.0",

src/unstract/sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.0"
+__version__ = "0.20.0"
 
 
 def get_sdk_version():

src/unstract/sdk/index.py

@@ -236,13 +236,17 @@ def index_file(
         full_text = []
         extracted_text = ""
         try:
-            x2text = X2Text(tool=self.tool)
-            x2text_adapter_inst: X2TextAdapter = x2text.get_x2text(
-                adapter_instance_id=x2text_adapter
-            )
-            extracted_text = x2text_adapter_inst.process(
-                input_file_path=file_path, output_file_path=output_file_path
-            )
+            if not output_file_path:
+                with open(file_path, encoding="utf-8") as file:
+                    extracted_text = file.read()
+            else:
+                x2text = X2Text(tool=self.tool)
+                x2text_adapter_inst: X2TextAdapter = x2text.get_x2text(
+                    adapter_instance_id=x2text_adapter
+                )
+                extracted_text = x2text_adapter_inst.process(
+                    input_file_path=file_path, output_file_path=output_file_path
+                )
         except AdapterError as e:
             # Wrapping AdapterErrors with SdkError
             raise IndexingError(str(e)) from e

src/unstract/sdk/llm.py

@@ -22,7 +22,7 @@
 class ToolLLM:
     """Class to handle LLMs for Unstract Tools."""
 
-    code_block_regex = re.compile(r"```.*?\n(.*?)\n```", re.DOTALL)
+    json_regex = re.compile(r"\{(?:.|\n)*\}")
 
     def __init__(
         self,
@@ -62,9 +62,9 @@ def run_completion(
         for i in range(retries):
             try:
                 response: CompletionResponse = llm.complete(prompt, **kwargs)
-                match = cls.code_block_regex.search(response.text)
+                match = cls.json_regex.search(response.text)
                 if match:
-                    response.text = match.group(1)
+                    response.text = match.group(0)
 
                 usage = {}
                 llm_token_counts = llm.callback_manager.handlers[