From d5a342d6e54c73417d625a18e9402afbe5282193 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M <117059509+chandrasekharan-zipstack@users.noreply.github.com> Date: Tue, 28 Jan 2025 10:34:33 +0530 Subject: [PATCH] feat: API changes for listing logs with file / level based filter support (#1083) * v2 API for listing logs with file / level based filter support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Commented code removed * Minor pre-commit fix and markdown text changes * refactor: Removed v2 API support for execution logs, using filters instead * Fixed org middleware behaviour for whitelisted paths --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Rahul Johny <116638720+johnyrahul@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- backend/backend/public_urls_v2.py | 3 --- backend/backend/settings/base.py | 7 ++++++- backend/file_management/constants.py | 6 ------ backend/middleware/organization_middleware.py | 7 +++---- .../workflow_manager/endpoint_v2/source.py | 7 ++++--- backend/workflow_manager/urls.py | 2 +- .../workflow_v2/execution_log_view.py | 21 +++++++++++++++---- .../workflow_v2/urls/__init__.py | 0 .../workflow_v2/{urls.py => urls/workflow.py} | 0 10 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 backend/workflow_manager/workflow_v2/urls/__init__.py rename backend/workflow_manager/workflow_v2/{urls.py => urls/workflow.py} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7792b3e33..6e80808bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,7 +80,7 @@ repos: --settings-path=pyproject.toml, ] - repo: https://github.com/hadialqattan/pycln - rev: v2.4.0 + rev: v2.5.0 hooks: - id: pycln args: [--config=pyproject.toml] diff --git a/backend/backend/public_urls_v2.py b/backend/backend/public_urls_v2.py index 366fc695f..bc1eca2ac 100644 --- a/backend/backend/public_urls_v2.py +++ b/backend/backend/public_urls_v2.py @@ -20,9 +20,6 @@ from django.conf.urls.static import static from django.urls import include, path -path_prefix = settings.PATH_PREFIX -api_path_prefix = settings.API_DEPLOYMENT_PATH_PREFIX - urlpatterns = [ path("", include("account_v2.urls")), # Connector OAuth diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index c49673d06..104ffc9f6 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -323,7 +323,7 @@ def get_required_setting( "middleware.cache_control.CacheControlMiddleware", ] -TENANT_SUBFOLDER_PREFIX = f"/{PATH_PREFIX}/unstract" +TENANT_SUBFOLDER_PREFIX = f"{PATH_PREFIX}/unstract" SHOW_PUBLIC_IF_NO_TENANT_FOUND = True TEMPLATES = [ @@ -435,6 +435,11 @@ def get_required_setting( "django_filters.rest_framework.DjangoFilterBackend", "rest_framework.filters.OrderingFilter", ], + # For API versioning + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", + "DEFAULT_VERSION": "v1", + "ALLOWED_VERSIONS": ["v1"], + "VERSION_PARAM": "version", } # These paths will work without authentication diff --git a/backend/file_management/constants.py b/backend/file_management/constants.py index a17b60b4c..b348066ad 100644 --- a/backend/file_management/constants.py +++ b/backend/file_management/constants.py @@ -6,9 +6,3 @@ class FileInformationKey: FILE_UPLOAD_MAX_SIZE = 100 * 1024 * 1024 FILE_UPLOAD_ALLOWED_EXT = ["pdf"] FILE_UPLOAD_ALLOWED_MIME = ["application/pdf"] - - -class FileViewTypes: - ORIGINAL = "ORIGINAL" - EXTRACT = "EXTRACT" - SUMMARIZE = "SUMMARIZE" diff --git a/backend/middleware/organization_middleware.py b/backend/middleware/organization_middleware.py index 848dd2c2c..5b0ffa605 100644 --- a/backend/middleware/organization_middleware.py +++ b/backend/middleware/organization_middleware.py @@ -6,8 +6,7 @@ class OrganizationMiddleware(MiddlewareMixin): def process_request(self, request): - tenant_prefix = settings.TENANT_SUBFOLDER_PREFIX.rstrip("/") + "/" - pattern = rf"^{tenant_prefix}(?P[^/]+)/" + pattern = r"^/api/(?Pv[12])/unstract/(?P[^/]+)/" # Check if the URL matches the pattern with organization ID match = re.match(pattern, request.path) @@ -17,12 +16,12 @@ def process_request(self, request): re.match(path, request.path) for path in settings.ORGANIZATION_MIDDLEWARE_WHITELISTED_PATHS ): - request.path_info = "/" + request.path_info return org_id = match.group("org_id") + version = match.group("version") request.organization_id = org_id - new_path = re.sub(pattern, "/" + tenant_prefix, request.path_info) + new_path = re.sub(pattern, f"/api/{version}/unstract/", request.path_info) request.path_info = new_path else: request.organization_id = None diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py index 194b321ce..633d4db6e 100644 --- a/backend/workflow_manager/endpoint_v2/source.py +++ b/backend/workflow_manager/endpoint_v2/source.py @@ -244,7 +244,7 @@ def publish_input_output_list_file_logs( return None folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders) - input_log = f"##Folders to process:\n\n{folders_list}\n\n" + input_log = f"## Folders to process:\n\n{folders_list}\n\n" self.execution_service.publish_update_log( state=LogState.INPUT_UPDATE, message=input_log ) @@ -256,9 +256,10 @@ def publish_input_output_list_file_logs( def publish_input_file_content(self, input_file_path: str, input_text: str) -> None: if not self.execution_service: return None - output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n" + output_log_message = f"## Input text:\n\n```text\n{input_text}\n```\n\n" input_log_message = ( - "##Input file:\n\n```text\n" f"{os.path.basename(input_file_path)}\n```\n\n" + "## Input file:\n\n```text\n" + f"{os.path.basename(input_file_path)}\n```\n\n" ) self.execution_service.publish_update_log( state=LogState.INPUT_UPDATE, message=input_log_message diff --git a/backend/workflow_manager/urls.py b/backend/workflow_manager/urls.py index 42c972973..43ee3ccb2 100644 --- a/backend/workflow_manager/urls.py +++ b/backend/workflow_manager/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path from workflow_manager.endpoint_v2 import urls as endpoint_urls -from workflow_manager.workflow_v2 import urls as workflow_urls +from workflow_manager.workflow_v2.urls import workflow as workflow_urls urlpatterns = [ path("endpoint/", include(endpoint_urls)), diff --git a/backend/workflow_manager/workflow_v2/execution_log_view.py b/backend/workflow_manager/workflow_v2/execution_log_view.py index 96f557b77..711789e8f 100644 --- a/backend/workflow_manager/workflow_v2/execution_log_view.py +++ b/backend/workflow_manager/workflow_v2/execution_log_view.py @@ -1,5 +1,6 @@ import logging +from django.db.models.query import QuerySet from permissions.permission import IsOwner from rest_framework import viewsets from rest_framework.versioning import URLPathVersioning @@ -16,12 +17,24 @@ class WorkflowExecutionLogViewSet(viewsets.ModelViewSet): serializer_class = WorkflowExecutionLogSerializer pagination_class = CustomPagination - EVENT_TIME_FELID_ASC = "event_time" + EVENT_TIME_FIELD_ASC = "event_time" - def get_queryset(self): + def get_queryset(self) -> QuerySet: # Get the execution_id:pk from the URL path execution_id = self.kwargs.get("pk") - queryset = ExecutionLog.objects.filter(execution_id=execution_id).order_by( - self.EVENT_TIME_FELID_ASC + filter_param = {"execution_id": execution_id} + + file_execution_id = self.request.query_params.get("file_execution_id") + if file_execution_id and file_execution_id == "null": + filter_param["file_execution_id"] = None + elif file_execution_id: + filter_param["file_execution_id"] = file_execution_id + + log_level = self.request.query_params.get("log_level") + if log_level: + filter_param["data__level"] = log_level.upper() + + queryset = ExecutionLog.objects.filter(**filter_param).order_by( + self.EVENT_TIME_FIELD_ASC ) return queryset diff --git a/backend/workflow_manager/workflow_v2/urls/__init__.py b/backend/workflow_manager/workflow_v2/urls/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/workflow_manager/workflow_v2/urls.py b/backend/workflow_manager/workflow_v2/urls/workflow.py similarity index 100% rename from backend/workflow_manager/workflow_v2/urls.py rename to backend/workflow_manager/workflow_v2/urls/workflow.py