Merge branch 'main' into pre-commit-ci-update-config

.python-version

@@ -1 +1 @@
-3.9.0
+3.9.6

CONTRIBUTE.md

@@ -15,7 +15,7 @@ Use LLMs to eliminate manual processes involving unstructured data.
 
 Just run the `run-platform.sh` launch script to get started in few minutes.
 
-The launch script does env setup with default values, pulls public docker images or builds them locally and finally runs them in containers.
+The launch script configures the env with sane defaults, pulls public docker images or builds them locally and finally runs them in containers.
 
 ```bash
 # Pull and run entire Unstract platform with default env config.
@@ -45,6 +45,7 @@ The launch script does env setup with default values, pulls public docker images
 
 Now visit [http://frontend.unstract.localhost](http://frontend.unstract.localhost) in your browser.
 
+NOTE: Modify the `.env` files present in each service folder to update its runtime behaviour. Run docker compose up again for the changes to take effect.```
 That's all. Enjoy!
 
 ## Authentication

README.md

@@ -84,7 +84,7 @@ Unstract comes well documented. You can get introduced to the [basics of Unstrac
 |<img src="docs/assets/3rd_party/weaviate.png" width="32"/>| Weaviate | ✅ Working |
 |<img src="docs/assets/3rd_party/pinecone.png" width="32"/>| Pinecone | ✅ Working |
 |<img src="docs/assets/3rd_party/postgres.png" width="32"/>| PostgreSQL | ✅ Working |
-|<img src="docs/assets/3rd_party/milvus.png" width="32"/>| Milvus | 🗓️ Coming soon! |
+|<img src="docs/assets/3rd_party/milvus.png" width="32"/>| Milvus | ✅ Working |
 
 
 
@@ -142,4 +142,4 @@ Contributions are welcome! Please read [CONTRIBUTE.md](CONTRIBUTE.md) for furthe
 
 ## 📊 A note on analytics
 
-In full disclosure, Unstract integrates Posthog to track usage analytics. As you can inspect the relevant code here, we collect the minimum possible metrics.
+In full disclosure, Unstract integrates Posthog to track usage analytics. As you can inspect the relevant code here, we collect the minimum possible metrics. Posthog can be disabled if desired by setting `REACT_APP_ENABLE_POSTHOG` to `false` in the frontend's .env file.

backend/.python-version

@@ -1 +1 @@
-3.9.0
+3.9.6

backend/account/authentication_controller.py

@@ -184,7 +184,9 @@ def set_user_organization(self, request: Request, organization_id: str) -> Respo
                         f"{ErrorMessage.ORGANIZATION_EXIST}, \
                             {ErrorMessage.DUPLICATE_API}"
                     )
-            self.create_tenant_user(organization=organization, user=user)
+            organization_member = self.create_tenant_user(
+                organization=organization, user=user
+            )
 
             if new_organization:
                 try:
@@ -209,6 +211,7 @@ def set_user_organization(self, request: Request, organization_id: str) -> Respo
             response: Response = Response(
                 status=status.HTTP_200_OK,
                 data={
+                    "is_new_org": new_organization,
                     "user": serialized_user_info,
                     "organization": organization_info,
                     f"{Common.LOG_EVENTS_ID}": StateStore.get(Common.LOG_EVENTS_ID),
@@ -221,6 +224,7 @@ def set_user_organization(self, request: Request, organization_id: str) -> Respo
                     organization_id=current_organization_id,
                 )
             UserSessionUtils.set_organization_id(request, organization_id)
+            UserSessionUtils.set_organization_member_role(request, organization_member)
             OrganizationMemberService.set_user_membership_in_organization_cache(
                 user_id=user.user_id, organization_id=organization_id
             )
@@ -419,12 +423,14 @@ def save_orgnanization_user_role(self, user_id: str, role: str) -> None:
             organization_user.role = role
             organization_user.save()
 
-    def create_tenant_user(self, organization: Organization, user: User) -> None:
+    def create_tenant_user(
+        self, organization: Organization, user: User
+    ) -> OrganizationMember:
         with tenant_context(organization):
             existing_tenant_user = OrganizationMemberService.get_user_by_id(id=user.id)
             if existing_tenant_user:
                 Logger.info(f"{existing_tenant_user.user.email} Already exist")
-
+                return existing_tenant_user
             else:
                 account_user = self.get_or_create_user(user=user)
                 if account_user:
@@ -441,7 +447,7 @@ def create_tenant_user(self, organization: Organization, user: User) -> None:
                         is_prompt_studio_onboarding_msg=False,
                     )
                     tenant_user.save()
-
+                    return tenant_user
                 else:
                     raise UserNotExistError()
 

backend/account/dto.py

@@ -59,6 +59,7 @@ class UserSessionInfo:
     email: str
     organization_id: str
     user: UserInfo
+    role: str
 
     @staticmethod
     def from_dict(data: dict[str, Any]) -> "UserSessionInfo":
@@ -67,6 +68,7 @@ def from_dict(data: dict[str, Any]) -> "UserSessionInfo":
             user_id=data["user_id"],
             email=data["email"],
             organization_id=data["organization_id"],
+            role=data["role"],
         )
 
     def to_dict(self) -> Any:
@@ -75,6 +77,7 @@ def to_dict(self) -> Any:
             "user_id": self.user_id,
             "email": self.email,
             "organization_id": self.organization_id,
+            "role": self.role,
         }
 
 

backend/account/serializer.py

@@ -116,3 +116,4 @@ class UserSessionResponseSerializer(serializers.Serializer):
     user_id = serializers.CharField()
     email = serializers.CharField()
     organization_id = serializers.CharField()
+    role = serializers.CharField()

backend/account/subscription_loader.py

@@ -4,6 +4,7 @@
 from typing import Any
 
 from django.apps import apps
+from django.utils import timezone
 
 logger = logging.getLogger(__name__)
 
@@ -75,3 +76,32 @@ def load_plugins() -> list[Any]:
         logger.info("No subscription plugins found.")
 
     return subscription_plugins
+
+
+def validate_etl_run(org_id: str) -> bool:
+    """Method to check subscription status before ETL runs.
+
+    Args:
+        org_id: The ID of the organization.
+
+    Returns:
+        A boolean indicating whether the pre-run check passed or not.
+    """
+    try:
+        from pluggable_apps.subscription.subscription_helper import SubscriptionHelper
+    except ModuleNotFoundError:
+        logger.error("Subscription plugin not found.")
+        return False
+
+    org_plans = SubscriptionHelper.get_subscription(org_id)
+    if not org_plans or not org_plans.is_active:
+        return False
+
+    if org_plans.is_paid:
+        return True
+
+    if timezone.now() >= org_plans.end_date:
+        logger.debug(f"Trial expired for org {org_id}")
+        return False
+
+    return True

backend/account/views.py

@@ -142,6 +142,7 @@ def make_session_response(
             email=request.user.email,
             user=auth_controller.get_user_info(request),
             organization_id=UserSessionUtils.get_organization_id(request),
+            role=UserSessionUtils.get_organization_member_role(request),
         )
     ).data
 

backend/adapter_processor/serializers.py

@@ -124,6 +124,10 @@ def to_representation(self, instance: AdapterInstance) -> dict[str, str]:
         rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(
             instance.adapter_id, common.ICON
         )
+        adapter_metadata = instance.get_adapter_meta_data()
+        model = adapter_metadata.get("model")
+        if model:
+            rep["model"] = model
 
         if instance.is_friction_less:
             rep["created_by_email"] = "Unstract"

backend/api/api_deployment_views.py

@@ -50,14 +50,17 @@ def post(
         serializer = ExecutionRequestSerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
         timeout = serializer.get_timeout(serializer.validated_data)
-
+        include_metadata = (
+            request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"
+        )
         if not file_objs or len(file_objs) == 0:
             raise InvalidAPIRequest("File shouldn't be empty")
         response = DeploymentHelper.execute_workflow(
             organization_name=org_name,
             api=api,
             file_objs=file_objs,
             timeout=timeout,
+            include_metadata=include_metadata,
         )
         if "error" in response and response["error"]:
             return Response(

backend/api/constants.py

@@ -1,6 +1,6 @@
 class ApiExecution:
     PATH: str = "deployment/api"
     MAXIMUM_TIMEOUT_IN_SEC: int = 300  # 5 minutes
-    DEFAULT_TIMEOUT_IN_SEC: int = 80
     FILES_FORM_DATA: str = "files"
     TIMEOUT_FORM_DATA: str = "timeout"
+    INCLUDE_METADATA: str = "include_metadata"

backend/api/deployment_helper.py

@@ -188,6 +188,7 @@ def execute_workflow(
         api: APIDeployment,
         file_objs: list[UploadedFile],
         timeout: int,
+        include_metadata: bool = False,
     ) -> ReturnDict:
         """Execute workflow by api.
 
@@ -215,6 +216,7 @@ def execute_workflow(
                 hash_values_of_files=hash_values_of_files,
                 timeout=timeout,
                 execution_id=execution_id,
+                include_metadata=include_metadata,
             )
             result.status_api = DeploymentHelper.construct_status_endpoint(
                 api_endpoint=api.api_endpoint, execution_id=execution_id

backend/api/postman_collection/dto.py

@@ -99,7 +99,12 @@ def create(
                 FormDataItem(
                     key=ApiExecution.TIMEOUT_FORM_DATA,
                     type="text",
-                    value=ApiExecution.DEFAULT_TIMEOUT_IN_SEC,
+                    value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,
+                ),
+                FormDataItem(
+                    key=ApiExecution.INCLUDE_METADATA,
+                    type="text",
+                    value=False,
                 ),
             ]
         )

backend/backend/constants.py

@@ -34,3 +34,4 @@ class FeatureFlag:
     """Temporary feature flags."""
 
     MULTI_TENANCY_V2 = "multi_tenancy_v2"
+    APP_DEPLOYMENT = "app_deployment"

backend/backend/settings/base.py

@@ -167,6 +167,7 @@ def get_required_setting(
     "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
 )
 
+INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
 # Flag to Enable django admin
 ADMIN_ENABLED = False
 

backend/backend/urls.py

@@ -29,7 +29,6 @@
     path("", include("file_management.urls")),
     path("", include("tool_instance.urls")),
     path("", include("pipeline.urls")),
-    path("", include("apps.urls")),
     path("", include("feature_flag.urls")),
     path("workflow/", include("workflow_manager.urls")),
     path("platform/", include("platform_settings.urls")),
@@ -62,11 +61,42 @@
     ),
 ]
 
+
+# APP deployment urls
+try:
+    import pluggable_apps.apps.app_deployment.urls  # noqa # pylint: disable=unused-import
+    import pluggable_apps.apps.canned_question.urls  # noqa # pylint: disable=unused-import
+    import pluggable_apps.apps.chat_history.urls  # noqa # pylint: disable=unused-import
+    import pluggable_apps.apps.chat_transcript.urls  # noqa # pylint: disable=unused-import
+
+    urlpatterns += [
+        path(
+            "canned_question/",
+            include("pluggable_apps.apps.canned_question.urls"),
+        ),
+        path("app/", include("pluggable_apps.apps.app_deployment.urls")),
+        path("chat_history/", include("pluggable_apps.apps.chat_history.urls")),
+        path("chat/", include("pluggable_apps.apps.chat_transcript.urls")),
+    ]
+except ImportError:
+    pass
+
+# Subscription urls
 try:
-    import pluggable_apps.subscription.urls  # noqa: F401
+
+    import pluggable_apps.subscription.urls  # noqa # pylint: disable=unused-import
 
     urlpatterns += [
         path("", include("pluggable_apps.subscription.urls")),
     ]
 except ImportError:
     pass
+
+try:
+    import pluggable_apps.manual_review.urls  # noqa: F401
+
+    urlpatterns += [
+        path("manual_review/", include("pluggable_apps.manual_review.urls")),
+    ]
+except ImportError:
+    pass

backend/connector/migrations/0003_alter_connectorinstance_connector_mode.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.1 on 2024-06-24 12:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("connector", "0002_connectorinstance_connector_metadata_b"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="connectorinstance",
+            name="connector_mode",
+            field=models.CharField(
+                choices=[
+                    (0, "UNKNOWN"),
+                    (1, "FILE_SYSTEM"),
+                    (2, "DATABASE"),
+                    (3, "APPDEPLOYMENT"),
+                ],
+                db_comment="Choices of connectors",
+                default=0,
+            ),
+        ),
+    ]