feat: API changes for listing logs with file / level based filter support

.pre-commit-config.yaml

@@ -80,7 +80,7 @@ repos:
             --settings-path=pyproject.toml,
           ]
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: pycln
         args: [--config=pyproject.toml]

backend/backend/base_urls.py

@@ -1,11 +1,12 @@
 # base_urls.py
 from django.conf import settings
-from django.urls import include, path
+from django.urls import include, path, re_path
 
 from .public_urls_v2 import urlpatterns as public_urls
 
 # Import urlpatterns from each file
 from .urls_v2 import urlpatterns as tenant_urls
+from .urls_v2 import urlpatterns_v2 as tenant_v2_urls
 
 # Combine the URL patterns
 urlpatterns = [
@@ -24,3 +25,11 @@
     ),
     path("", include("health.urls")),
 ]
+
+# TODO: Add versioning regex to PATH_PREFIX and handle version kwarg for view functions
+urlpatterns += [
+    re_path(
+        r"^api/(?P<version>(v1|v2))/unstract/",
+        include((tenant_v2_urls, "tenant_v2"), namespace="tenant_v2"),
+    ),
+]

backend/backend/public_urls_v2.py

@@ -20,9 +20,6 @@
 from django.conf.urls.static import static
 from django.urls import include, path
 
-path_prefix = settings.PATH_PREFIX
-api_path_prefix = settings.API_DEPLOYMENT_PATH_PREFIX
-
 urlpatterns = [
     path("", include("account_v2.urls")),
     # Connector OAuth

backend/backend/settings/base.py

@@ -324,7 +324,7 @@ def get_required_setting(
     "middleware.cache_control.CacheControlMiddleware",
 ]
 
-TENANT_SUBFOLDER_PREFIX = f"/{PATH_PREFIX}/unstract"
+TENANT_SUBFOLDER_PREFIX = f"{PATH_PREFIX}/unstract"
 SHOW_PUBLIC_IF_NO_TENANT_FOUND = True
 
 TEMPLATES = [
@@ -432,6 +432,11 @@ def get_required_setting(
     "DEFAULT_PERMISSION_CLASSES": [],  # TODO: Update once auth is figured
     "TEST_REQUEST_DEFAULT_FORMAT": "json",
     "EXCEPTION_HANDLER": "middleware.exception.drf_logging_exc_handler",
+    # For API versioning
+    "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning",
+    "DEFAULT_VERSION": "v1",
+    "ALLOWED_VERSIONS": ["v1", "v2"],
+    "VERSION_PARAM": "version",
 }
 
 # These paths will work without authentication

backend/backend/urls_v2.py

@@ -59,3 +59,7 @@
         include("prompt_studio.prompt_studio_index_manager_v2.urls"),
     ),
 ]
+
+urlpatterns_v2 = [
+    path("workflow/", include("workflow_manager.workflow_v2.urls.execution_log")),
+]

backend/file_management/constants.py

@@ -6,9 +6,3 @@ class FileInformationKey:
     FILE_UPLOAD_MAX_SIZE = 100 * 1024 * 1024
     FILE_UPLOAD_ALLOWED_EXT = ["pdf"]
     FILE_UPLOAD_ALLOWED_MIME = ["application/pdf"]
-
-
-class FileViewTypes:
-    ORIGINAL = "ORIGINAL"
-    EXTRACT = "EXTRACT"
-    SUMMARIZE = "SUMMARIZE"

backend/middleware/organization_middleware.py

@@ -1,20 +1,19 @@
 import re
 
-from django.conf import settings
 from django.utils.deprecation import MiddlewareMixin
 
 
 class OrganizationMiddleware(MiddlewareMixin):
     def process_request(self, request):
-        tenant_prefix = settings.TENANT_SUBFOLDER_PREFIX.rstrip("/") + "/"
-        pattern = rf"^{tenant_prefix}(?P<org_id>[^/]+)/"
+        pattern = r"^/api/(?P<version>v[12])/unstract/(?P<org_id>[^/]+)/"
 
         # Check if the URL matches the pattern with organization ID
         match = re.match(pattern, request.path)
         if match:
             org_id = match.group("org_id")
+            version = match.group("version")
             request.organization_id = org_id
-            new_path = re.sub(pattern, "/" + tenant_prefix, request.path_info)
+            new_path = re.sub(pattern, f"/api/{version}/unstract/", request.path_info)
             request.path_info = new_path
         else:
             request.organization_id = None

backend/workflow_manager/endpoint_v2/source.py

@@ -243,7 +243,7 @@ def publish_input_output_list_file_logs(
             return None
 
         folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)
-        input_log = f"##Folders to process:\n\n{folders_list}\n\n"
+        input_log = f"## Folders to process:\n\n{folders_list}\n\n"
         self.execution_service.publish_update_log(
             state=LogState.INPUT_UPDATE, message=input_log
         )
@@ -255,9 +255,10 @@ def publish_input_output_list_file_logs(
     def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
         if not self.execution_service:
             return None
-        output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"
+        output_log_message = f"## Input text:\n\n```text\n{input_text}\n```\n\n"
         input_log_message = (
-            "##Input file:\n\n```text\n" f"{os.path.basename(input_file_path)}\n```\n\n"
+            "## Input file:\n\n```text\n"
+            f"{os.path.basename(input_file_path)}\n```\n\n"
         )
         self.execution_service.publish_update_log(
             state=LogState.INPUT_UPDATE, message=input_log_message

backend/workflow_manager/urls.py

@@ -1,6 +1,6 @@
 from django.urls import include, path
 from workflow_manager.endpoint_v2 import urls as endpoint_urls
-from workflow_manager.workflow_v2 import urls as workflow_urls
+from workflow_manager.workflow_v2.urls import workflow as workflow_urls
 
 urlpatterns = [
     path("endpoint/", include(endpoint_urls)),

backend/workflow_manager/workflow_v2/execution_log_view.py

@@ -1,5 +1,6 @@
 import logging
 
+from django.db.models.query import QuerySet
 from permissions.permission import IsOwner
 from rest_framework import viewsets
 from rest_framework.versioning import URLPathVersioning
@@ -13,15 +14,27 @@
 class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):
     versioning_class = URLPathVersioning
     permission_classes = [IsOwner]
+    allowed_versions = ["v1", "v2"]
     serializer_class = WorkflowExecutionLogSerializer
     pagination_class = CustomPagination
 
-    EVENT_TIME_FELID_ASC = "event_time"
+    EVENT_TIME_FIELD_ASC = "event_time"
 
-    def get_queryset(self):
+    def get_queryset(self) -> QuerySet:
         # Get the execution_id:pk from the URL path
         execution_id = self.kwargs.get("pk")
-        queryset = ExecutionLog.objects.filter(execution_id=execution_id).order_by(
-            self.EVENT_TIME_FELID_ASC
+        filter_param = {"execution_id": execution_id}
+
+        # In v2, allow filtering by file_execution_id and log level
+        if self.request.version == "v2":
+            file_execution_id = self.request.query_params.get("file_execution_id")
+            filter_param["file_execution_id"] = file_execution_id
+
+            log_level = self.request.query_params.get("log_level")
+            if log_level:
+                filter_param["data__level"] = log_level.upper()
+
+        queryset = ExecutionLog.objects.filter(**filter_param).order_by(
+            self.EVENT_TIME_FIELD_ASC
         )
         return queryset

backend/workflow_manager/workflow_v2/urls/execution_log.py

@@ -0,0 +1,15 @@
+from django.urls import path
+from rest_framework.urlpatterns import format_suffix_patterns
+from workflow_manager.workflow_v2.execution_log_view import WorkflowExecutionLogViewSet
+
+execution_log_list = WorkflowExecutionLogViewSet.as_view({"get": "list"})
+
+urlpatterns = format_suffix_patterns(
+    [
+        path(
+            "execution/<uuid:pk>/logs/",
+            execution_log_list,
+            name="execution-log",
+        ),
+    ]
+)

backend/workflow_manager/workflow_v2/urls.py → backend/workflow_manager/workflow_v2/urls/workflow.py

@@ -1,6 +1,5 @@
 from django.urls import path
 from rest_framework.urlpatterns import format_suffix_patterns
-from workflow_manager.workflow_v2.execution_log_view import WorkflowExecutionLogViewSet
 from workflow_manager.workflow_v2.execution_view import WorkflowExecutionViewSet
 from workflow_manager.workflow_v2.views import WorkflowViewSet
 
@@ -23,7 +22,6 @@
 workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})
 execution_entity = WorkflowExecutionViewSet.as_view({"get": "retrieve"})
 execution_list = WorkflowExecutionViewSet.as_view({"get": "list"})
-execution_log_list = WorkflowExecutionLogViewSet.as_view({"get": "list"})
 workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})
 workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})
 workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})
@@ -63,11 +61,6 @@
             execution_entity,
             name="workflow-detail",
         ),
-        path(
-            "execution/<uuid:pk>/logs/",
-            execution_log_list,
-            name="execution-log",
-        ),
         path(
             "schema/",
             workflow_schema,