From 535a2c28aa101830bcbd05ac88e284fc57fc8f44 Mon Sep 17 00:00:00 2001 From: Rahul Johny <116638720+johnyrahul@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:59:07 +0530 Subject: [PATCH] Handing API deployment errors (#507) * Handing API deployment errors * Handing API deployment errors * Updated tool version to refer latest sdk * added null check on result --- backend/sample.env | 4 ++-- tools/classifier/requirements.txt | 2 +- tools/classifier/src/config/properties.json | 2 +- tools/structure/requirements.txt | 2 +- tools/structure/src/config/properties.json | 2 +- tools/text_extractor/requirements.txt | 2 +- .../text_extractor/src/config/properties.json | 2 +- .../unstract/tool_registry/public_tools.json | 12 ++++++------ .../workflow_execution/tools_utils.py | 3 +-- .../workflow_execution/workflow_execution.py | 4 +++- worker/src/unstract/worker/constants.py | 7 +++++++ worker/src/unstract/worker/exception.py | 4 ++++ worker/src/unstract/worker/worker.py | 19 ++++++++++++++++++- 13 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 worker/src/unstract/worker/exception.py diff --git a/backend/sample.env b/backend/sample.env index f605c45b6..cda1e835f 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data # Structure Tool Image (Runs prompt studio exported tools) # https://hub.docker.com/r/unstract/tool-structure -STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.31" +STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.32" STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure" -STRUCTURE_TOOL_IMAGE_TAG="0.0.31" +STRUCTURE_TOOL_IMAGE_TAG="0.0.32" # Feature Flags EVALUATION_SERVER_IP=unstract-flipt diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txt index 2a30fa417..225c3d2c0 100644 --- a/tools/classifier/requirements.txt +++ b/tools/classifier/requirements.txt @@ -1,4 +1,4 @@ # Add your dependencies here # Required for all unstract tools -unstract-sdk~=0.38.0 +unstract-sdk~=0.38.1 diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json index c86c089d9..a46b2c91c 100644 --- a/tools/classifier/src/config/properties.json +++ b/tools/classifier/src/config/properties.json @@ -2,7 +2,7 @@ "schemaVersion": "0.0.1", "displayName": "File Classifier", "functionName": "classify", - "toolVersion": "0.0.27", + "toolVersion": "0.0.28", "description": "Classifies a file into a bin based on its contents", "input": { "description": "File to be classified" diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt index 2a30fa417..225c3d2c0 100644 --- a/tools/structure/requirements.txt +++ b/tools/structure/requirements.txt @@ -1,4 +1,4 @@ # Add your dependencies here # Required for all unstract tools -unstract-sdk~=0.38.0 +unstract-sdk~=0.38.1 diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json index 386d897d0..0e7400f0a 100644 --- a/tools/structure/src/config/properties.json +++ b/tools/structure/src/config/properties.json @@ -2,7 +2,7 @@ "schemaVersion": "0.0.1", "displayName": "Structure Tool", "functionName": "structure_tool", - "toolVersion": "0.0.31", + "toolVersion": "0.0.32", "description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio", "input": { "description": "File that needs to be indexed and parsed for answers" diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txt index 2a30fa417..225c3d2c0 100644 --- a/tools/text_extractor/requirements.txt +++ b/tools/text_extractor/requirements.txt @@ -1,4 +1,4 @@ # Add your dependencies here # Required for all unstract tools -unstract-sdk~=0.38.0 +unstract-sdk~=0.38.1 diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json index 5a85d1462..dcca4ca14 100644 --- a/tools/text_extractor/src/config/properties.json +++ b/tools/text_extractor/src/config/properties.json @@ -2,7 +2,7 @@ "schemaVersion": "0.0.1", "displayName": "Text Extractor", "functionName": "text_extractor", - "toolVersion": "0.0.25", + "toolVersion": "0.0.26", "description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents", "input": { "description": "Document" diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json index 729679652..e09e4aaed 100644 --- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json +++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json @@ -5,7 +5,7 @@ "schemaVersion": "0.0.1", "displayName": "File Classifier", "functionName": "classify", - "toolVersion": "0.0.27", + "toolVersion": "0.0.28", "description": "Classifies a file into a bin based on its contents", "input": { "description": "File to be classified" @@ -139,9 +139,9 @@ } }, "icon": "\n\n \n \n \n \n \n \n \n \n \n \n \n\n", - "image_url": "docker:unstract/tool-classifier:0.0.27", + "image_url": "docker:unstract/tool-classifier:0.0.28", "image_name": "unstract/tool-classifier", - "image_tag": "0.0.27" + "image_tag": "0.0.28" }, "text_extractor": { "tool_uid": "text_extractor", @@ -149,7 +149,7 @@ "schemaVersion": "0.0.1", "displayName": "Text Extractor", "functionName": "text_extractor", - "toolVersion": "0.0.25", + "toolVersion": "0.0.26", "description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents", "input": { "description": "Document" @@ -224,8 +224,8 @@ } }, "icon": "\n\n \n \n \n \n \n \n \n \n \n \n \n\n", - "image_url": "docker:unstract/tool-text-extractor:0.0.25", + "image_url": "docker:unstract/tool-text-extractor:0.0.26", "image_name": "unstract/tool-text-extractor", - "image_tag": "0.0.25" + "image_tag": "0.0.26" } } diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py index b138c4089..d5fe76804 100644 --- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py +++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py @@ -185,8 +185,7 @@ def run_tool_with_retry( try: response = tool_sandbox.run_tool() if response: - result: dict[str, Any] = response.get("result", {}) - return result + return response logger.warning( f"ToolExecutionException - Retrying " f"({retry_count + 1}/{max_retries})" diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py index 1f8bce13d..270ebb670 100644 --- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py +++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py @@ -201,9 +201,11 @@ def _execute_step( message="Ready for execution", component=tool_instance_id, ) - self.tool_utils.run_tool( + result = self.tool_utils.run_tool( tool_sandbox=sandbox, ) + if result and result.get("error"): + raise ToolOutputNotFoundException(result.get("error")) if not self.validate_execution_result(step + 1): raise ToolOutputNotFoundException( f"Tool exception raised for tool {tool_uid}, " diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py index 9a6e02810..92c3a7b11 100644 --- a/worker/src/unstract/worker/constants.py +++ b/worker/src/unstract/worker/constants.py @@ -6,6 +6,13 @@ class LogType: SINGLE_STEP = "SINGLE_STEP_MESSAGE" +class LogLevel: + ERROR = "ERROR" + WARN = "WARN" + INFO = "INFO" + DEBUG = "DEBUG" + + class ToolKey: TOOL_INSTANCE_ID = "tool_instance_id" diff --git a/worker/src/unstract/worker/exception.py b/worker/src/unstract/worker/exception.py new file mode 100644 index 000000000..c7c639877 --- /dev/null +++ b/worker/src/unstract/worker/exception.py @@ -0,0 +1,4 @@ +class ToolRunException(Exception): + def __init__(self, message): + self.message = message + super().__init__(self.message) diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py index d62a058bd..30abbfe5a 100644 --- a/worker/src/unstract/worker/worker.py +++ b/worker/src/unstract/worker/worker.py @@ -11,7 +11,8 @@ ContainerClientInterface, ContainerInterface, ) -from unstract.worker.constants import Env, LogType, ToolKey +from unstract.worker.constants import Env, LogLevel, LogType, ToolKey +from unstract.worker.exception import ToolRunException from unstract.core.constants import LogFieldName from unstract.core.pubsub_helper import LogPublisher @@ -79,6 +80,9 @@ def process_log_message( if not log_dict: return None log_type = self.get_log_type(log_dict) + log_level = self.get_log_level(log_dict) + if log_type == LogType.LOG and log_level == LogLevel.ERROR: + raise ToolRunException(log_dict.get("log")) if not self.is_valid_log_type(log_type): self.logger.warning( f"Received invalid logType: {log_type} with log message: {log_dict}" @@ -114,6 +118,12 @@ def get_log_type(self, log_dict: Any) -> Optional[str]: return log_type return None + def get_log_level(self, log_dict: Any) -> Optional[str]: + if isinstance(log_dict, dict): + log_level: Optional[str] = log_dict.get("level") + return log_level + return None + def run_command(self, command: str) -> Optional[Any]: """Runs any given command on the container. @@ -210,6 +220,13 @@ def run_container( execution_id=execution_id, organization_id=organization_id, ) + except ToolRunException as te: + self.logger.error( + f"Error while running docker container: {te}", + stack_info=True, + exc_info=True, + ) + result = {"type": "RESULT", "result": None, "error": str(te.message)} except Exception as e: self.logger.error( f"Failed to run docker container: {e}", stack_info=True, exc_info=True