Merge branch 'main' into feat/beckrock-embeddings-integration

.gitignore

@@ -636,6 +636,7 @@ docker/*.env
 !docker/sample*.env
 docker/public_tools.json
 docker/proxy_overrides.yaml
+docker/compose.override.yaml
 docker/workflow_data/
 
 # Tool development

backend/README.md

@@ -39,13 +39,21 @@ pdm install --dev -G lint
 pdm install --prod --no-editable
 ```
 
+#### Running scripts
+
 PDM allows you to run scripts applicable within the service dir.
 
 ```bash
 # List the possible scripts that can be executed
 pdm run -l
 ```
 
+For example to run the backend (dev mode is recommended to take advantage of gunicorn's `reload` feature)
+
+```bash
+pdm run backend --dev
+```
+
 #### Running commands
 
 - If you plan to run the django server locally, make sure the dependent services are up (either locally or through docker compose)

backend/api_v2/api_deployment_views.py

@@ -4,7 +4,7 @@
 
 from api_v2.constants import ApiExecution
 from api_v2.deployment_helper import DeploymentHelper
-from api_v2.exceptions import InvalidAPIRequest, NoActiveAPIKeyError
+from api_v2.exceptions import NoActiveAPIKeyError
 from api_v2.models import APIDeployment
 from api_v2.postman_collection.dto import PostmanCollection
 from api_v2.serializers import (
@@ -47,16 +47,14 @@ def initialize_request(
     def post(
         self, request: Request, org_name: str, api_name: str, api: APIDeployment
     ) -> Response:
-        file_objs = request.FILES.getlist(ApiExecution.FILES_FORM_DATA)
         serializer = ExecutionRequestSerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
+        file_objs = serializer.validated_data.get(ApiExecution.FILES_FORM_DATA)
         timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)
         include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)
         include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)
         use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)
         tag_names = serializer.validated_data.get(ApiExecution.TAGS)
-        if not file_objs or len(file_objs) == 0:
-            raise InvalidAPIRequest("File shouldn't be empty")
         response = DeploymentHelper.execute_workflow(
             organization_name=org_name,
             api=api,

backend/api_v2/key_helper.py

@@ -4,6 +4,7 @@
 from api_v2.exceptions import UnauthorizedKey
 from api_v2.models import APIDeployment, APIKey
 from api_v2.serializers import APIKeySerializer
+from django.core.exceptions import ValidationError
 from pipeline_v2.models import Pipeline
 from rest_framework.request import Request
 from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
@@ -29,7 +30,7 @@ def validate_api_key(
             api_key_instance: APIKey = APIKey.objects.get(api_key=api_key)
             if not KeyHelper.has_access(api_key_instance, instance):
                 raise UnauthorizedKey()
-        except APIKey.DoesNotExist:
+        except (APIKey.DoesNotExist, ValidationError):
             raise UnauthorizedKey()
 
     @staticmethod

backend/api_v2/serializers.py

@@ -9,8 +9,10 @@
 from rest_framework.serializers import (
     BooleanField,
     CharField,
+    FileField,
     IntegerField,
     JSONField,
+    ListField,
     ModelSerializer,
     Serializer,
     ValidationError,
@@ -115,12 +117,31 @@ class ExecutionRequestSerializer(TagParamsSerializer):
             e.g:'tag1,tag2-name,tag3_name'
     """
 
+    MAX_FILES_ALLOWED = 32
+
     timeout = IntegerField(
         min_value=-1, max_value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC, default=-1
     )
     include_metadata = BooleanField(default=False)
     include_metrics = BooleanField(default=False)
     use_file_history = BooleanField(default=False)
+    files = ListField(
+        child=FileField(),
+        required=True,
+        allow_empty=False,
+        error_messages={
+            "required": "At least one file must be provided.",
+            "empty": "The file list cannot be empty.",
+        },
+    )
+
+    def validate_files(self, value):
+        """Validate the files field."""
+        if len(value) == 0:
+            raise ValidationError("The file list cannot be empty.")
+        if len(value) > self.MAX_FILES_ALLOWED:
+            raise ValidationError(f"Maximum '{self.MAX_FILES_ALLOWED}' files allowed.")
+        return value
 
 
 class ExecutionQuerySerializer(Serializer):

backend/backend/wsgi.py

@@ -6,7 +6,9 @@
 https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
 """
 
+import logging
 import os
+import time
 
 from django.conf import settings
 from django.core.wsgi import get_wsgi_application
@@ -15,11 +17,17 @@
 
 load_dotenv()
 
+logger = logging.getLogger(__name__)
+
 os.environ.setdefault(
     "DJANGO_SETTINGS_MODULE",
     os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
 )
 
+wsgi_start_time = time.perf_counter()
 django_app = get_wsgi_application()
+wsgi_init_elapsed = time.perf_counter() - wsgi_start_time
+logger.info(f"WSGI application initialized in {wsgi_init_elapsed:.3f} seconds")
+
 
 application = start_server(django_app, f"{settings.PATH_PREFIX}/socket")

backend/entrypoint.sh

@@ -1,19 +1,50 @@
 #!/usr/bin/env bash
 
-cmd=$1
-if [ "$cmd" = "migrate" ]; then
+show_help() {
+    echo "Usage: ./entrypoint.sh [OPTIONS]"
+    echo ""
+    echo "Options:"
+    echo "  --migrate        Perform database migrations before starting the server."
+    echo "  --dev            Run Gunicorn in development mode with --reload and reduced graceful timeout (5s)."
+    echo "  --help, -h       Show this help message and exit."
+}
+
+# Parse arguments
+migrate=false
+dev=false
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --migrate) migrate=true ;;
+        --dev) dev=true ;;
+        --help|-h) show_help; exit 0 ;;
+        *) echo "Unknown argument: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+# Perform database migration if --migrate is provided
+if [ "$migrate" = true ]; then
     echo "Migration initiated"
     .venv/bin/python manage.py migrate
 fi
 
-# NOTE: Leaving below for reference incase required in the future
-# python manage.py runserver 0.0.0.0:8000 --insecure
-# NOTE updated socket threads
-.venv/bin/gunicorn \
-    --bind 0.0.0.0:8000 \
-    --workers 2 \
-    --threads 2 \
-    --log-level debug \
-    --timeout 600 \
-    --access-logfile - \
-    backend.wsgi:application
+# Configure Gunicorn based on --dev flag
+gunicorn_args=(
+    --bind 0.0.0.0:8000
+    --workers 2
+    --threads 2
+    --log-level debug
+    --timeout 600
+    --access-logfile -
+)
+
+if [ "$dev" = true ]; then
+    echo "Running in development mode"
+    gunicorn_args+=(--reload --graceful-timeout 5)
+else
+    echo "Running in production mode"
+fi
+
+# Start Gunicorn
+.venv/bin/gunicorn "${gunicorn_args[@]}" backend.wsgi:application