Merge branch 'main' into fix/shared_tool_export_fix

backend/backend/public_urls.py

@@ -48,12 +48,19 @@
 
 try:
     import pluggable_apps.platform_admin.urls  # noqa # pylint: disable=unused-import
-    import pluggable_apps.public_shares_share_controller.urls  # noqa # pylint: disable=unused-import
+
+    urlpatterns += [
+        path(f"{path_prefix}/", include("pluggable_apps.platform_admin.urls")),
+    ]
+except ImportError:
+    pass
+
+try:
+    import pluggable_apps.public_shares.share_controller.urls  # noqa # pylint: disable=unused-import
 
     share_path_prefix = settings.PUBLIC_PATH_PREFIX
 
     urlpatterns += [
-        path(f"{path_prefix}/", include("pluggable_apps.platform_admin.urls")),
         # Public Sharing
         path(
             f"{share_path_prefix}/",

backend/connector_processor/connector_processor.py

@@ -12,8 +12,7 @@
     InValidConnectorId,
     InValidConnectorMode,
     OAuthTimeOut,
-    TestConnectorException,
-    TestConnectorInputException,
+    TestConnectorInputError,
 )
 
 from unstract.connectors.base import UnstractConnector
@@ -100,15 +99,15 @@ def get_all_supported_connectors(
         return supported_connectors
 
     @staticmethod
-    def test_connectors(connector_id: str, cred_string: dict[str, Any]) -> bool:
+    def test_connectors(connector_id: str, credentials: dict[str, Any]) -> bool:
         logger.info(f"Testing connector: {connector_id}")
         connector: dict[str, Any] = fetch_connectors_by_key_value(
             ConnectorKeys.ID, connector_id
         )[0]
         if connector.get(ConnectorKeys.OAUTH):
             try:
-                oauth_key = cred_string.get(ConnectorAuthKey.OAUTH_KEY)
-                cred_string = ConnectorAuthHelper.get_oauth_creds_from_cache(
+                oauth_key = credentials.get(ConnectorAuthKey.OAUTH_KEY)
+                credentials = ConnectorAuthHelper.get_oauth_creds_from_cache(
                     cache_key=oauth_key, delete_key=False
                 )
             except Exception as exc:
@@ -120,17 +119,13 @@ def test_connectors(connector_id: str, cred_string: dict[str, Any]) -> bool:
 
         try:
             connector_impl = Connectorkit().get_connector_by_id(
-                connector_id, cred_string
+                connector_id, credentials
             )
             test_result = connector_impl.test_credentials()
             logger.info(f"{connector_id} test result: {test_result}")
             return test_result
         except ConnectorError as e:
-            logger.error(f"Error while testing {connector_id}: {e}")
-            raise TestConnectorInputException(core_err=e)
-        except Exception as e:
-            logger.error(f"Error while testing {connector_id}: {e}")
-            raise TestConnectorException
+            raise TestConnectorInputError(core_err=e)
 
     def get_connector_data_with_key(connector_id: str, key_value: str) -> Any:
         """Generic Function to get connector data with provided key."""

backend/connector_processor/exceptions.py

@@ -31,7 +31,7 @@ class JSONParseException(APIException):
 
 class OAuthTimeOut(APIException):
     status_code = 408
-    default_detail = "Timed Out. Please re authenticate."
+    default_detail = "Timed out. Please re-authenticate."
 
 
 class InternalServiceError(APIException):
@@ -44,7 +44,7 @@ class TestConnectorException(APIException):
     default_detail = "Error while testing connector."
 
 
-class TestConnectorInputException(UnstractBaseException):
+class TestConnectorInputError(UnstractBaseException):
     def __init__(self, core_err: ConnectorError) -> None:
         super().__init__(detail=core_err.message, core_err=core_err)
         self.default_detail = core_err.message

backend/connector_processor/views.py

@@ -67,10 +67,10 @@ def test(self, request: Request) -> Response:
         """Tests the connector against the credentials passed."""
         serializer: TestConnectorSerializer = self.get_serializer(data=request.data)
         serializer.is_valid(raise_exception=True)
-        connector_id = serializer.validated_data.get(ConnectorKeys.CONNECTOR_ID)
+        connector_id = serializer.validated_data.get(CIKey.CONNECTOR_ID)
         cred_string = serializer.validated_data.get(CIKey.CONNECTOR_METADATA)
         test_result = ConnectorProcessor.test_connectors(
-            connector_id=connector_id, cred_string=cred_string
+            connector_id=connector_id, credentials=cred_string
         )
         return Response(
             {ConnectorKeys.IS_VALID: test_result},

backend/tool_instance/serializers.py

@@ -68,6 +68,11 @@ def create(self, validated_data: dict[str, Any]) -> Any:
             raise ValidationError(f"Workflow with ID {workflow_id} does not exist.")
         validated_data[TIKey.WORKFLOW] = workflow
 
+        if workflow.workflow_tool.count() > 0:
+            raise ValidationError(
+                f"Workflow with ID {workflow_id} can't have more than one tool."
+            )
+
         tool_uid = validated_data.get(TIKey.TOOL_ID)
         if not tool_uid:
             raise ToolDoesNotExist()

backend/workflow_manager/endpoint/constants.py

@@ -70,9 +70,9 @@ class FileType:
 
 
 class FilePattern:
-    PDF_DOCUMENTS = ["*.pdf"]
+    PDF_DOCUMENTS = ["*.pdf", "*.PDF"]
     TEXT_DOCUMENTS = ["*.txt"]
-    IMAGES = ["*.jpg", "*.jpeg", "*.png", "*.gif", "*.bmp"]
+    IMAGES = ["*.jpg", "*.jpeg", "*.png", "*.gif", "*.bmp", "*.tif", "*.tiff"]
 
 
 class SourceConstant:

backend/workflow_manager/workflow/generator.py

@@ -17,6 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
+# TODO: Can be removed as not getting used with UX chnages.
 class WorkflowGenerator:
     """Helps with generating a workflow using the LLM."""
 

backend/workflow_manager/workflow/views.py

@@ -4,6 +4,7 @@
 from connector.connector_instance_helper import ConnectorInstanceHelper
 from django.conf import settings
 from django.db.models.query import QuerySet
+from numpy import deprecate_with_doc
 from permissions.permission import IsOwner
 from pipeline.models import Pipeline
 from pipeline.pipeline_processor import PipelineProcessor
@@ -78,6 +79,7 @@ def get_serializer_class(self) -> serializers.Serializer:
         else:
             return WorkflowSerializer
 
+    @deprecate_with_doc("Not using with the latest UX chnages")
     def _generate_workflow(self, workflow_id: str) -> WorkflowGenerator:
         registry_tools: list[Tool] = ToolProcessor.get_registry_tools()
         generator = WorkflowGenerator(workflow_id=workflow_id)
@@ -86,18 +88,12 @@ def _generate_workflow(self, workflow_id: str) -> WorkflowGenerator:
         return generator
 
     def perform_update(self, serializer: WorkflowSerializer) -> Workflow:
-        """To edit a workflow. Regenerates the tool instances for a new prompt.
+        """To edit a workflow.
 
         Raises: WorkflowGenerationError
         """
         kwargs = {}
-        if serializer.validated_data.get(WorkflowKey.PROMPT_TEXT):
-            workflow: Workflow = self.get_object()
-            generator = self._generate_workflow(workflow_id=workflow.id)
-            kwargs = {
-                WorkflowKey.LLM_RESPONSE: generator.llm_response,
-                WorkflowKey.WF_IS_ACTIVE: True,
-            }
+
         try:
             workflow = serializer.save(**kwargs)
             return workflow

docker/scripts/merge_env.py

@@ -54,8 +54,8 @@ def _merge_to_env_file(base_env_file_path: str, target_env: dict[str, str] = {})
     target env.
 
     Args:
-        base_env_path (string): Base env file path.
-        target_env (dict, optional): Target env to use for merge.
+        base_env_file_path (string): Base env file path e.g. `sample.env`
+        target_env (dict, optional): Target env to use for merge e.g. `.env`
 
     Returns:
         string: File contents after merge.

frontend/src/components/agency/workflow-execution-layout/WorkflowExecutionMain.jsx

@@ -1,7 +1,6 @@
 import { Col, Row } from "antd";
 import PropTypes from "prop-types";
 
-import { Prompt } from "../prompt/Prompt";
 import { Steps } from "../steps/Steps";
 import "./WorkflowExecutionMain.css";
 import { InputOutput } from "../input-output/InputOutput";
@@ -19,9 +18,6 @@ function WorkflowExecutionMain({
       <Row className="wf-exec-main-layout">
         <Col span={8} className="wf-exec-main-layout">
           <div className="wf-exec-main-col-1">
-            <div className="wf-exec-main-prompt">
-              <Prompt />
-            </div>
             <div className="wf-exec-main-steps">
               <Steps
                 setSteps={setSteps}

run-platform.sh

@@ -69,7 +69,7 @@ display_help() {
   echo -e "   -e, --only-env      Only do env files setup"
   echo -e "   -p, --only-pull     Only do docker images pull"
   echo -e "   -b, --build-local   Build docker images locally"
-  echo -e "   -u, --upgrade       Upgrade services"
+  echo -e "   -u, --update        Update services version"
   echo -e "   -x, --trace         Enables trace mode"
   echo -e "   -V, --verbose       Print verbose logs"
   echo -e "   -v, --version       Docker images version tag (default \"latest\")"
@@ -93,8 +93,8 @@ parse_args() {
       -b | --build-local)
         opt_build_local=true
         ;;
-      -u | --upgrade)
-        opt_upgrade=true
+      -u | --update)
+        opt_update=true
         ;;
       -x | --trace)
         set -o xtrace  # display every line before execution; enables PS4
@@ -126,21 +126,40 @@ parse_args() {
   debug "OPTION only_env: $opt_only_env"
   debug "OPTION only_pull: $opt_only_pull"
   debug "OPTION build_local: $opt_build_local"
-  debug "OPTION upgrade: $opt_upgrade"
+  debug "OPTION upgrade: $opt_update"
   debug "OPTION verbose: $opt_verbose"
   debug "OPTION version: $opt_version"
 }
 
 do_git_pull() {
-  if [ "$opt_upgrade" = false ]; then
+  if [ "$opt_update" = false ]; then
     return
   fi
 
-  echo -e "Performing git switch to ""$blue_text""main branch""$default_text".
-  git switch main
+  echo "Fetching release tags."
+  git fetch --quiet --tags
 
-  echo -e "Performing ""$blue_text""git pull""$default_text"" on main branch."
-  git pull
+  if [[ "$opt_version" == "latest" ]]; then
+    branch=`git describe --tags --abbrev=0`
+  elif [[ "$opt_version" == "main" ]]; then
+    branch="main"
+    opt_build_local=true
+    echo -e "Choosing ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""main""$default_text"" branch."
+  elif [ -z $(git tag -l "$opt_version") ]; then
+    echo -e "$red_text""Version not found.""$default_text"
+    if [[ ! $opt_version == v* ]]; then
+      echo -e "$red_text""Version must be provided with a 'v' prefix (e.g. v0.47.0).""$default_text"
+    fi
+    exit 1
+  else
+    branch="$opt_version"
+  fi
+
+  echo -e "Performing ""$blue_text""git checkout""$default_text"" to ""$blue_text""$branch""$default_text""."
+  git checkout --quiet $branch
+
+  echo -e "Performing ""$blue_text""git pull""$default_text"" on ""$blue_text""$branch""$default_text""."
+  git pull --quiet $(git remote) $branch
 }
 
 setup_env() {
@@ -180,7 +199,7 @@ setup_env() {
         fi
       fi
       echo -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."
-    elif [ "$opt_upgrade" = true ]; then
+    elif [ "$opt_update" = true ]; then
       python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path
       if [ $? -ne 0 ]; then
         exit 1
@@ -192,7 +211,7 @@ setup_env() {
   if [ ! -e "$script_dir/docker/essentials.env" ]; then
     cp "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
     echo -e "Created env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
-  elif [ "$opt_upgrade" = true ]; then
+  elif [ "$opt_update" = true ]; then
     python3 $script_dir/docker/scripts/merge_env.py "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
     if [ $? -ne 0 ]; then
       exit 1
@@ -202,9 +221,9 @@ setup_env() {
 
   # Not part of an upgrade.
   if [ ! -e "$script_dir/docker/proxy_overrides.yaml" ]; then
-    echo -e "NOTE: Proxy behaviour can be overridden via ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text""."
+    echo -e "NOTE: Reverse proxy config can be overridden via ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text""."
   else
-    echo -e "Found ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text"". Proxy behaviour will be overridden."
+    echo -e "Found ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text"". ""$yellow_text""Reverse proxy config will be overridden.""$default_text"
   fi
 
   if [ "$opt_only_env" = true ]; then
@@ -221,7 +240,7 @@ build_services() {
       echo -e "$red_text""Failed to build docker images.""$default_text"
       exit 1
     }
-  elif [ "$first_setup" = true ] || [ "$opt_upgrade" = true ]; then
+  elif [ "$first_setup" = true ] || [ "$opt_update" = true ]; then
     echo -e "$blue_text""Pulling""$default_text"" docker images tag ""$blue_text""$opt_version""$default_text""."
     # Try again on a slow network.
     VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.yaml pull ||
@@ -246,15 +265,19 @@ run_services() {
   echo -e "$blue_text""Starting docker containers in detached mode""$default_text"
   VERSION=$opt_version $docker_compose_cmd up -d
 
-  if [ "$opt_upgrade" = true ]; then
+  if [ "$opt_update" = true ]; then
     echo ""
-    echo -e "$green_text""Upgraded platform to $opt_version version.""$default_text"
+    if [[ "$opt_version" == "main" ]]; then
+      echo -e "$green_text""Updated platform to latest main (unstable).""$default_text"
+    else
+      echo -e "$green_text""Updated platform to $opt_version version.""$default_text"
+    fi
   fi
   echo -e "\nOnce the services are up, visit ""$blue_text""http://frontend.unstract.localhost""$default_text"" in your browser."
-  echo "See logs with:"
+  echo -e "\nSee logs with:"
   echo -e "    ""$blue_text""$docker_compose_cmd -f docker/docker-compose.yaml logs -f""$default_text"
-  echo -e "Configure services by updating their ""$yellow_text"".env""$default_text"" files."
-  echo "Make sure to restart the services with:"
+  echo -e "Configure services by updating corresponding ""$yellow_text""<service>/.env""$default_text"" files."
+  echo -e "Make sure to ""$yellow_text""restart""$default_text"" the services with:"
   echo -e "    ""$blue_text""$docker_compose_cmd -f docker/docker-compose.yaml up -d""$default_text"
 
   popd 1>/dev/null
@@ -268,7 +291,7 @@ check_dependencies
 opt_only_env=false
 opt_only_pull=false
 opt_build_local=false
-opt_upgrade=false
+opt_update=false
 opt_verbose=false
 opt_version="latest"
 

unstract/connectors/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "s3fs[boto3]==2023.6.0", # For Minio
     "PyDrive2[fsspec]==1.15.4", # For GDrive
     "oauth2client==4.1.3", # For GDrive
-    "dropboxdrivefs==1.3.1", # For Dropbox
+    "dropboxdrivefs==1.4.1", # For Dropbox
     "boxfs==0.2.1", # For Box
     "gcsfs==2023.6.0", # For GoogleCloudStorage
     "adlfs==2023.8.0", # For AzureCloudStorage

unstract/connectors/src/unstract/connectors/databases/unstract_db.py

@@ -68,7 +68,7 @@ def test_credentials(self) -> bool:
         try:
             self.get_engine()
         except Exception as e:
-            raise ConnectorError(str(e)) from e
+            raise ConnectorError(f"Error while connecting to DB: {str(e)}") from e
         return True
 
     def execute(self, query: str) -> Any:

unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py

@@ -65,7 +65,11 @@ def get_fsspec_fs(self) -> AzureBlobFileSystem:
     def test_credentials(self) -> bool:
         """To test credentials for Azure Cloud Storage."""
         try:
-            self.get_fsspec_fs().ls(f"{self.bucket}")
+            is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))
+            if not is_dir:
+                raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")
         except Exception as e:
-            raise ConnectorError(str(e))
+            raise ConnectorError(
+                f"Error from Azure Cloud Storage while testing connection: {str(e)}"
+            ) from e
         return True

unstract/connectors/src/unstract/connectors/filesystems/box/box.py

@@ -24,7 +24,7 @@ def __init__(self, settings: dict[str, Any]):
                 settings_dict = json.loads(settings["box_app_settings"])
                 if not isinstance(settings_dict, dict):
                     raise ConnectorError(
-                        "Box app settings is expected to be a valid JSON",
+                        "Box app settings should be a valid JSON.",
                         treat_as_user_message=True,
                     )
             except JSONDecodeError as e:
@@ -112,8 +112,15 @@ def get_fsspec_fs(self) -> BoxFileSystem:
 
     def test_credentials(self) -> bool:
         """To test credentials for the Box connector."""
+        is_dir = False
         try:
-            self.get_fsspec_fs().isdir("/")
+            is_dir = bool(self.get_fsspec_fs().isdir("/"))
         except Exception as e:
-            raise ConnectorError(str(e))
+            raise ConnectorError(
+                f"Error from Box while testing connection: {str(e)}"
+            ) from e
+        if not is_dir:
+            raise ConnectorError(
+                "Unable to connect to Box, please check the connection settings."
+            )
         return True