feat: Bumped SDK version to 0.50.0 in services and tools (#768)

* Bumped SDK version to 0.50.0 in services and tools, update platform service API to fetch adapter information * Commit pdm.lock changes * Bumped SDK version to 0.50.0 in prompt-service * Commit pdm.lock changes * Commit pdm.lock changes --------- Co-authored-by: Ritwik G <[email protected]> Co-authored-by: ritwik-g <[email protected]> Co-authored-by: chandrasekharan-zipstack <[email protected]> Co-authored-by: ali <[email protected]> Co-authored-by: muhammad-ali-e <[email protected]>
Zipstack · Oct 7, 2024 · 5c4f9bc · 5c4f9bc
1 parent 844b239
commit 5c4f9bc
Show file tree

Hide file tree

Showing 19 changed files with 613 additions and 492 deletions.
diff --git a/backend/pdm.lock b/backend/pdm.lock
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import time
 import uuid
 from pathlib import Path
 from typing import Any, Callable, Optional
@@ -338,6 +339,7 @@ def index_document(
             )
             file_path = str(Path(file_path) / file_name)
 
+        start_time = time.time()
         logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")
         PromptStudioHelper._publish_log(
             {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
@@ -367,12 +369,16 @@ def index_document(
             process_text=process_text,
         )
 
-        logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")
+        elapsed_time = time.time() - start_time
+        logger.info(
+            f"[{tool_id}] Indexing successful for doc: {file_name},"
+            f" took {elapsed_time:.3f}s"
+        )
         PromptStudioHelper._publish_log(
             {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
             LogLevels.INFO,
             LogLevels.RUN,
-            "Indexing successful",
+            f"Indexing successful, took {elapsed_time:.3f}s",
         )
 
         return doc_id.get("output")

diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import time
 import uuid
 from pathlib import Path
 from typing import Any, Callable, Optional
@@ -343,6 +344,7 @@ def index_document(
             logger.error(f"No tool instance found for the ID {tool_id}")
             raise ToolNotValid()
 
+        start_time = time.time()
         logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")
         PromptStudioHelper._publish_log(
             {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
@@ -372,12 +374,17 @@ def index_document(
             process_text=process_text,
         )
 
+        elapsed_time = time.time() - start_time
+        logger.info(
+            f"[{tool_id}] Indexing successful for doc: {file_name},"
+            f" took {elapsed_time:.3f}s"
+        )
         logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")
         PromptStudioHelper._publish_log(
             {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
             LogLevels.INFO,
             LogLevels.RUN,
-            "Indexing successful",
+            f"Indexing successful, took {elapsed_time:.3f}s",
         )
 
         return doc_id.get("output")

diff --git a/backend/pyproject.toml b/backend/pyproject.toml
@@ -32,7 +32,7 @@ dependencies = [
     "python-socketio==5.9.0", # For log_events
     "social-auth-app-django==5.3.0", # For OAuth
     "social-auth-core==4.4.2", # For OAuth
-    "unstract-sdk~=0.49.0",
+    "unstract-sdk~=0.50.0",
     # ! IMPORTANT!
     # Indirect local dependencies usually need to be added in their own projects
     # as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.

diff --git a/pdm.lock b/pdm.lock
diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.py
@@ -28,21 +28,23 @@ def get_adapter_instance_from_db(
         """
         if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
             query = (
-                "SELECT id, adapter_id, adapter_metadata_b FROM "
-                f'"{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x '
+                "SELECT id, adapter_id, adapter_name, adapter_type, adapter_metadata_b"
+                f' FROM "{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x '
                 f"WHERE id='{adapter_instance_id}' and "
                 f"organization_id='{organization_uid}'"
             )
         else:
             query = (
-                f"SELECT id, adapter_id, adapter_metadata_b FROM "
-                f'"{organization_id}".adapter_adapterinstance x '
+                f"SELECT id, adapter_id, adapter_name, adapter_type, adapter_metadata_b"
+                f' FROM "{organization_id}".adapter_adapterinstance x '
                 f"WHERE id='{adapter_instance_id}'"
             )
         cursor = db.execute_sql(query)
         result_row = cursor.fetchone()
         if not result_row:
-            raise APIError(message="Adapter not found", code=404)
+            raise APIError(
+                message=f"Adapter '{adapter_instance_id}' not found", code=404
+            )
         columns = [desc[0] for desc in cursor.description]
         data_dict: dict[str, Any] = dict(zip(columns, result_row))
         cursor.close()

diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
     "flask~=3.0",
     "llama-index==0.10.58",
     "python-dotenv==1.0.0",
-    "unstract-sdk~=0.49.0",
+    "unstract-sdk~=0.50.0",
     "redis>=5.0.3",
     "unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",
     "unstract-flags @ file:///${PROJECT_ROOT}/../unstract/flags",

diff --git a/pyproject.toml b/pyproject.toml
@@ -41,7 +41,7 @@ hook-check-django-migrations = [
     "psycopg2-binary==2.9.9",
     "python-dotenv==1.0.0",
     "python-magic==0.4.27",
-    "unstract-sdk~=0.49.0",
+    "unstract-sdk~=0.50.0",
     "-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
     "-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",
     "-e unstract-flags @ file:///${PROJECT_ROOT}/unstract/flags",

diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txt
@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.49.0
+unstract-sdk~=0.50.0
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
   "schemaVersion": "0.0.1",
   "displayName": "File Classifier",
   "functionName": "classify",
-  "toolVersion": "0.0.35",
+  "toolVersion": "0.0.36",
   "description": "Classifies a file into a bin based on its contents",
   "input": {
     "description": "File to be classified"

diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.49.0
+unstract-sdk~=0.50.0
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
   "schemaVersion": "0.0.1",
   "displayName": "Structure Tool",
   "functionName": "structure_tool",
-  "toolVersion": "0.0.42",
+  "toolVersion": "0.0.44",
   "description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
   "input": {
     "description": "File that needs to be indexed and parsed for answers"

diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
@@ -13,10 +13,18 @@
 from unstract.sdk.utils import ToolUtils
 from unstract.sdk.utils.common_utils import CommonUtils
 
+PAID_FEATURE_MSG = (
+    "It is a cloud / enterprise feature. If you have purchased a plan and still "
+    "face this issue, please contact support"
+)
+
 
 class StructureTool(BaseTool):
     def validate(self, input_file: str, settings: dict[str, Any]) -> None:
-        pass
+        enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
+        challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")
+        if enable_challenge and not challenge_llm:
+            raise ValueError("Challenge LLM is not set after enabling Challenge")
 
     def run(
         self,
@@ -25,12 +33,14 @@ def run(
         output_dir: str,
     ) -> None:
         prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
-        challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]
-        enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]
-        summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]
-        single_pass_extraction_mode: bool = settings[
-            SettingsKeys.SINGLE_PASS_EXTRACTION_MODE
-        ]
+        enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
+        summarize_as_source: bool = settings.get(
+            SettingsKeys.SUMMARIZE_AS_SOURCE, False
+        )
+        single_pass_extraction_mode: bool = settings.get(
+            SettingsKeys.SINGLE_PASS_EXTRACTION_MODE, False
+        )
+        challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")
         responder: PromptTool = PromptTool(
             tool=self,
             prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT),
@@ -42,7 +52,7 @@ def run(
                 tool=self, prompt_registry_id=prompt_registry_id
             )
             tool_metadata = exported_tool[SettingsKeys.TOOL_METADATA]
-            self.stream_log(f"Tool Metadata retrived succesfully: {tool_metadata}")
+            self.stream_log(f"Tool metadata retrieved successfully: {tool_metadata}")
         except Exception as e:
             self.stream_error_and_exit(f"Error loading structure definition: {e}")
 
@@ -84,18 +94,20 @@ def run(
         }
         # TODO: Need to split extraction and indexing
         # to avoid unwanted indexing
-        self.stream_log("Indexing document")
+        source_file_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)
+        self.stream_log(f"Indexing document '{source_file_name}'")
         usage_kwargs: dict[Any, Any] = dict()
         usage_kwargs[SettingsKeys.RUN_ID] = run_id
-        usage_kwargs[SettingsKeys.FILE_NAME] = (
-            self.get_exec_metadata.get(MetadataKey.SOURCE_NAME),
-        )
+        usage_kwargs[SettingsKeys.FILE_NAME] = source_file_name
 
         process_text = None
         try:
             from helper import process_text  # type: ignore [attr-defined]
         except ImportError:
-            self.stream_log("Function 'process_text' is not found")
+            self.stream_log(
+                f"Function to higlight context is not found. {PAID_FEATURE_MSG}",
+                level=LogLevel.WARN,
+            )
 
         if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
             index.index(
@@ -186,7 +198,7 @@ def run(
                     # To check if the prompt has table enforce type selected.
                     pass
 
-            self.stream_log("Fetching responses for prompts...")
+            self.stream_log(f"Fetching responses for {len(outputs)} prompt(s)...")
             prompt_service_resp = responder.answer_prompt(
                 payload=payload,
             )
@@ -211,7 +223,10 @@ def run(
                     highlight_data = transform_dict(epilogue, tool_data_dir)
                     metadata[SettingsKeys.HIGHLIGHT_DATA] = highlight_data
                 except ImportError:
-                    self.stream_log("Function 'transform_dict' is not found")
+                    self.stream_log(
+                        f"Highlight metadata is not added. {PAID_FEATURE_MSG}",
+                        level=LogLevel.WARN,
+                    )
             # Update the dictionary with modified metadata
             structured_output_dict[SettingsKeys.METADATA] = metadata
             structured_output = json.dumps(structured_output_dict)

diff --git a/tools/text_extractor/README.md b/tools/text_extractor/README.md
@@ -112,6 +112,3 @@ docker run -it \
     --log-level DEBUG
 
 ```
-
-License
-This project is licensed under the Zipstack Inc.
diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txt
@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.49.0
+unstract-sdk~=0.50.0
diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json
@@ -2,7 +2,7 @@
   "schemaVersion": "0.0.1",
   "displayName": "Text Extractor",
   "functionName": "text_extractor",
-  "toolVersion": "0.0.33",
+  "toolVersion": "0.0.34",
   "description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
   "input": {
     "description": "Document"

diff --git a/unstract/tool-registry/pyproject.toml b/unstract/tool-registry/pyproject.toml
@@ -13,7 +13,7 @@ dependencies = [
     "docker~=6.1.3",
     "jsonschema~=4.18.2",
     "PyYAML~=6.0.1",
-    "unstract-sdk~=0.49.0",
+    "unstract-sdk~=0.50.0",
     # ! IMPORTANT!
     # Local dependencies usually need to be added as:
     # https://pdm-project.org/latest/usage/dependency/#local-dependencies

diff --git a/worker/sample.env b/worker/sample.env
@@ -14,7 +14,6 @@ LOG_LEVEL="INFO"
 
 # Flag to decide whether to clean up/ remove the tool container after execution.
 # (Default: True)
-
 REMOVE_CONTAINER_ON_EXIT=True
 
 # Client module path of the container engine to be used.
-Original file line number
+Diff line change
@@ Expand Up / @@ -112,6 +112,3 @@ docker run -it \ @@
         --log-level DEBUG
     ```
-    License
-    This project is licensed under the Zipstack Inc.