diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 9b9deea2f..67cb99f15 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -210,6 +210,8 @@ def get_required_setting( "prompt_studio.prompt_studio_core", "prompt_studio.prompt_studio_registry", "prompt_studio.prompt_studio_output_manager", + "prompt_studio.prompt_studio_document_manager", + "prompt_studio.prompt_studio_index_manager", ) INSTALLED_APPS = list(SHARED_APPS) + [ diff --git a/backend/backend/urls.py b/backend/backend/urls.py index 505d279c7..fc5681b70 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -50,4 +50,12 @@ UrlPathConstants.PROMPT_STUDIO, include("prompt_studio.prompt_studio_output_manager.urls"), ), + path( + UrlPathConstants.PROMPT_STUDIO, + include("prompt_studio.prompt_studio_document_manager.urls"), + ), + path( + UrlPathConstants.PROMPT_STUDIO, + include("prompt_studio.prompt_studio_index_manager.urls"), + ), ] diff --git a/backend/file_management/constants.py b/backend/file_management/constants.py index b348066ad..8ada83316 100644 --- a/backend/file_management/constants.py +++ b/backend/file_management/constants.py @@ -6,3 +6,9 @@ class FileInformationKey: FILE_UPLOAD_MAX_SIZE = 100 * 1024 * 1024 FILE_UPLOAD_ALLOWED_EXT = ["pdf"] FILE_UPLOAD_ALLOWED_MIME = ["application/pdf"] + +class FileViewTypes: + ORIGINAL = "ORIGINAL" + EXTRACT = "EXTRACT" + SUMMARIZE = "SUMMARIZE" + diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py index 3b5032101..257d8d839 100644 --- a/backend/file_management/file_management_helper.py +++ b/backend/file_management/file_management_helper.py @@ -194,7 +194,8 @@ def fetch_file_contents( elif file_content_type == "text/plain": with fs.open(file_path, "r") as file: - FileManagerHelper.logger.info(f"Reading text file: {file_path}") + FileManagerHelper.logger.info( + f"Reading text file: {file_path}") text_content = file.read() return text_content else: diff --git a/backend/file_management/serializer.py b/backend/file_management/serializer.py index 110dbd1ff..e18679185 100644 --- a/backend/file_management/serializer.py +++ b/backend/file_management/serializer.py @@ -50,8 +50,9 @@ class FileUploadIdeSerializer(serializers.Serializer): class FileInfoIdeSerializer(serializers.Serializer): - file_name = serializers.CharField() + document_id = serializers.CharField() tool_id = serializers.CharField() + view_type = serializers.CharField(required=False) class FileListRequestIdeSerializer(serializers.Serializer): diff --git a/backend/file_management/views.py b/backend/file_management/views.py index 06bb74977..b494f0168 100644 --- a/backend/file_management/views.py +++ b/backend/file_management/views.py @@ -4,6 +4,7 @@ from connector.models import ConnectorInstance from django.http import HttpRequest +from file_management.constants import FileViewTypes from file_management.exceptions import ( ConnectorInstanceNotFound, ConnectorOAuthError, @@ -20,10 +21,15 @@ FileUploadSerializer, ) from oauth2client.client import HttpAccessTokenRefreshError +from prompt_studio.prompt_studio_document_manager.models import DocumentManager +from prompt_studio.prompt_studio_document_manager.prompt_studio_document_helper import ( + PromptStudioDocumentHelper, +) from rest_framework import serializers, status, viewsets from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning + from unstract.connectors.exceptions import ConnectorError from unstract.connectors.filesystems.local_storage.local_storage import ( LocalStorageFS, @@ -132,8 +138,21 @@ def upload_for_ide(self, request: HttpRequest) -> Response: tool_id=tool_id, ) file_system = LocalStorageFS(settings={"path": file_path}) + + documents = [] for uploaded_file in uploaded_files: file_name = uploaded_file.name + + # Create a record in the db for the file + document = PromptStudioDocumentHelper.create( + tool_id=tool_id, document_name=file_name) + # Create a dictionary to store document data + doc = { + "document_id": document.document_id, + "document_name": document.document_name, + "tool": document.tool.tool_id + } + # Store file logger.info( f"Uploading file: {file_name}" if file_name @@ -145,14 +164,31 @@ def upload_for_ide(self, request: HttpRequest) -> Response: uploaded_file, file_name, ) - return Response({"message": "Files are uploaded successfully!"}) + documents.append(doc) + return Response({"data": documents}) @action(detail=True, methods=["get"]) def fetch_contents_ide(self, request: HttpRequest) -> Response: serializer = FileInfoIdeSerializer(data=request.GET) serializer.is_valid(raise_exception=True) - file_name: str = serializer.validated_data.get("file_name") + document_id: str = serializer.validated_data.get("document_id") + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + file_name: str = document.document_name tool_id: str = serializer.validated_data.get("tool_id") + view_type: str = serializer.validated_data.get("view_type") + + filename_without_extension = file_name.rsplit('.', 1)[0] + if view_type == FileViewTypes.EXTRACT: + file_name = ( + f"{FileViewTypes.EXTRACT.lower()}/" + f"{filename_without_extension}.txt" + ) + if view_type == FileViewTypes.SUMMARIZE: + file_name = ( + f"{FileViewTypes.SUMMARIZE.lower()}/" + f"{filename_without_extension}.txt" + ) + file_path = ( file_path ) = FileManagerHelper.handle_sub_directory_for_tenants( @@ -165,7 +201,8 @@ def fetch_contents_ide(self, request: HttpRequest) -> Response: if not file_path.endswith("/"): file_path += "/" file_path += file_name - contents = FileManagerHelper.fetch_file_contents(file_system, file_path) + contents = FileManagerHelper.fetch_file_contents( + file_system, file_path) return Response({"data": contents}, status=status.HTTP_200_OK) @action(detail=True, methods=["get"]) @@ -196,7 +233,9 @@ def list_ide(self, request: HttpRequest) -> Response: def delete(self, request: HttpRequest) -> Response: serializer = FileInfoIdeSerializer(data=request.GET) serializer.is_valid(raise_exception=True) - file_name: str = serializer.validated_data.get("file_name") + document_id: str = serializer.validated_data.get("document_id") + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + file_name: str = document.document_name tool_id: str = serializer.validated_data.get("tool_id") file_path = FileManagerHelper.handle_sub_directory_for_tenants( request.org_id, @@ -205,13 +244,12 @@ def delete(self, request: HttpRequest) -> Response: tool_id=tool_id, ) path = file_path - if not file_name: - return Response( - {"data": "File deletion failed. File name is mandatory"}, - status=status.HTTP_400_BAD_REQUEST, - ) file_system = LocalStorageFS(settings={"path": path}) try: + # Delete the document record + document.delete() + + # Delete the file FileManagerHelper.delete_file(file_system, path, file_name) return Response( {"data": "File deleted succesfully."}, diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py index a00397d52..2f2634fc4 100644 --- a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py +++ b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py @@ -11,6 +11,14 @@ class Migration(migrations.Migration): "prompt_profile_manager", "0007_profilemanager_is_default_and_more", ), + ( + "prompt_studio", + "0006_alter_toolstudioprompt_prompt_key_and_more", + ), + ( + "prompt_studio_core", + "0007_remove_customtool_default_profile_and_more", + ) ] def MigrateProfileManager(apps: Any, schema_editor: Any) -> None: diff --git a/backend/prompt_studio/prompt_profile_manager/views.py b/backend/prompt_studio/prompt_profile_manager/views.py index aae07fbb3..f070f8975 100644 --- a/backend/prompt_studio/prompt_profile_manager/views.py +++ b/backend/prompt_studio/prompt_profile_manager/views.py @@ -45,7 +45,7 @@ def get_queryset(self) -> Optional[QuerySet]: def create( self, request: HttpRequest, *args: tuple[Any], **kwargs: dict[str, Any] ) -> Response: - serializer = self.get_serializer(data=request.data) + serializer: ProfileManagerSerializer = self.get_serializer(data=request.data) # Overriding default exception behaviour # TO DO : Handle model related exceptions. serializer.is_valid(raise_exception=True) diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py index 6c5b4ad01..2bc8e8889 100644 --- a/backend/prompt_studio/prompt_studio_core/constants.py +++ b/backend/prompt_studio/prompt_studio_core/constants.py @@ -77,6 +77,7 @@ class ToolStudioPromptKeys: EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed" SUMMARIZE = "summarize" SUMMARIZED_RESULT = "summarized_result" + DOCUMENT_ID = "document_id" class LogLevels: diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py index bd5e1aab7..90e10c7fc 100644 --- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py +++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py @@ -23,6 +23,9 @@ from prompt_studio.prompt_studio_core.prompt_ide_base_tool import ( PromptIdeBaseTool, ) +from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import ( + PromptStudioIndexHelper, +) from unstract.sdk.constants import LogLevel from unstract.sdk.index import ToolIndex from unstract.sdk.prompt import PromptTool @@ -85,6 +88,7 @@ def index_document( file_name: str, org_id: str, user_id: str, + document_id: str, is_summary: bool = False, ) -> Any: """Method to index a document. @@ -149,6 +153,7 @@ def index_document( tool_id=tool_id, file_name=file_path, org_id=org_id, + document_id=document_id, is_summary=is_summary, ) logger.info(f"Indexing done sucessfully for {file_name}") @@ -164,7 +169,12 @@ def index_document( @staticmethod def prompt_responder( - id: str, tool_id: str, file_name: str, org_id: str, user_id: str + id: str, + tool_id: str, + file_name: str, + org_id: str, + user_id: str, + document_id: str ) -> Any: """Execute chain/single run of the prompts. Makes a call to prompt service and returns the dict of response. @@ -217,7 +227,8 @@ def prompt_responder( ), ) if not prompt_instance: - logger.error(f"Prompt id {id} does not have any data in db") + logger.error( + f"Prompt id {id} does not have any data in db") raise PromptNotValid() except Exception as exc: logger.error(f"Error while fetching prompt {exc}") @@ -242,7 +253,11 @@ def prompt_responder( ) logger.info(f"Invoking prompt service for prompt id {id}") response = PromptStudioHelper._fetch_response( - path=file_path, tool=tool, prompts=prompts, org_id=org_id + path=file_path, + tool=tool, + prompts=prompts, + org_id=org_id, + document_id=document_id ) stream_log.publish( tool.tool_id, @@ -262,6 +277,7 @@ def _fetch_response( path: str, prompts: list[ToolStudioPrompt], org_id: str, + document_id: str ) -> Any: """Utility function to invoke prompt service. Used internally. @@ -302,6 +318,7 @@ def _fetch_response( file_name=path, tool_id=str(tool.tool_id), org_id=org_id, + document_id=document_id, is_summary=tool.summarize_as_source, ) @@ -382,6 +399,7 @@ def dynamic_indexer( tool_id: str, file_name: str, org_id: str, + document_id: str, is_summary: bool = False, ) -> str: try: @@ -400,7 +418,7 @@ def dynamic_indexer( extract_file_path = os.path.join( directory, "extract", os.path.splitext(filename)[0] + ".txt" ) - return str( + doc_id = str( tool_index.index_file( tool_id=tool_id, embedding_type=embedding_model, @@ -414,3 +432,12 @@ def dynamic_indexer( output_file_path=extract_file_path, ) ) + + PromptStudioIndexHelper.handle_index_manager( + document_id=document_id, + is_summary=is_summary, + profile_manager=profile_manager, + doc_id=doc_id, + ) + + return doc_id diff --git a/backend/prompt_studio/prompt_studio_core/serializers.py b/backend/prompt_studio/prompt_studio_core/serializers.py index f711e1230..87f8d90b2 100644 --- a/backend/prompt_studio/prompt_studio_core/serializers.py +++ b/backend/prompt_studio/prompt_studio_core/serializers.py @@ -62,7 +62,7 @@ def to_representation(self, instance): # type: ignore class PromptStudioIndexSerializer(serializers.Serializer): - file_name = serializers.CharField() + document_id = serializers.CharField() tool_id = serializers.CharField() diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py index a81203344..0f090f7c8 100644 --- a/backend/prompt_studio/prompt_studio_core/views.py +++ b/backend/prompt_studio/prompt_studio_core/views.py @@ -24,6 +24,7 @@ from prompt_studio.prompt_studio_core.prompt_studio_helper import ( PromptStudioHelper, ) +from prompt_studio.prompt_studio_document_manager.models import DocumentManager from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.request import Request @@ -196,15 +197,17 @@ def index_document(self, request: HttpRequest) -> Response: tool_id: str = serializer.validated_data.get( ToolStudioPromptKeys.TOOL_ID ) - file_name: str = serializer.validated_data.get( - ToolStudioPromptKeys.FILE_NAME - ) + document_id: str = serializer.validated_data.get( + ToolStudioPromptKeys.DOCUMENT_ID) + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + file_name: str = document.document_name try: unique_id = PromptStudioHelper.index_document( tool_id=tool_id, file_name=file_name, org_id=request.org_id, user_id=request.user.user_id, + document_id=document_id, ) for processor_plugin in self.processor_plugins: @@ -216,6 +219,7 @@ def index_document(self, request: HttpRequest) -> Response: file_name=file_name, org_id=request.org_id, user_id=request.user.user_id, + document_id=document_id, ) if unique_id: @@ -246,7 +250,9 @@ def fetch_response(self, request: HttpRequest) -> Response: Response """ tool_id: str = request.data.get(ToolStudioPromptKeys.TOOL_ID) - file_name: str = request.data.get(ToolStudioPromptKeys.FILE_NAME) + document_id: str = request.data.get(ToolStudioPromptKeys.DOCUMENT_ID) + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + file_name: str = document.document_name id: str = request.data.get(ToolStudioPromptKeys.ID) if not file_name or file_name == ToolStudioPromptKeys.UNDEFINED: @@ -258,5 +264,6 @@ def fetch_response(self, request: HttpRequest) -> Response: file_name=file_name, org_id=request.org_id, user_id=request.user.user_id, + document_id=document_id, ) return Response(response, status=status.HTTP_200_OK) diff --git a/backend/prompt_studio/prompt_studio_document_manager/__init__.py b/backend/prompt_studio/prompt_studio_document_manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/prompt_studio/prompt_studio_document_manager/admin.py b/backend/prompt_studio/prompt_studio_document_manager/admin.py new file mode 100644 index 000000000..a16311f1b --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import DocumentManager + +admin.site.register(DocumentManager) diff --git a/backend/prompt_studio/prompt_studio_document_manager/apps.py b/backend/prompt_studio/prompt_studio_document_manager/apps.py new file mode 100644 index 000000000..90bd8928b --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PromptStudioDocumentManagerConfig(AppConfig): + name = 'prompt_studio.prompt_studio_document_manager' diff --git a/backend/prompt_studio/prompt_studio_document_manager/constants.py b/backend/prompt_studio/prompt_studio_document_manager/constants.py new file mode 100644 index 000000000..2ac55ca83 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/constants.py @@ -0,0 +1,4 @@ +class PSDMKeys: + DOCUMENT_NAME = "document_name" + TOOL = "tool" + DOCUMENT_ID = "document_id" diff --git a/backend/prompt_studio/prompt_studio_document_manager/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_document_manager/migrations/0001_initial.py new file mode 100644 index 000000000..f926cdcaf --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/migrations/0001_initial.py @@ -0,0 +1,77 @@ +# Generated by Django 4.2.1 on 2024-03-10 20:48 + +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("prompt_studio_core", "0007_remove_customtool_default_profile_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="DocumentManager", + fields=[ + ("created_at", models.DateTimeField(auto_now_add=True)), + ("modified_at", models.DateTimeField(auto_now=True)), + ( + "document_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "document_name", + models.CharField( + db_comment="Field to store the document name", editable=False + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="prompt_document_created_by", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="prompt_document_modified_by", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "tool", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_document_linked_tool", + to="prompt_studio_core.customtool", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="documentmanager", + constraint=models.UniqueConstraint( + fields=("document_name", "tool"), name="unique_document_name_tool" + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_document_manager/migrations/__init__.py b/backend/prompt_studio/prompt_studio_document_manager/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/prompt_studio/prompt_studio_document_manager/models.py b/backend/prompt_studio/prompt_studio_document_manager/models.py new file mode 100644 index 000000000..f45ed3830 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/models.py @@ -0,0 +1,55 @@ +import uuid + +from account.models import User +from django.db import models +from prompt_studio.prompt_studio_core.models import CustomTool +from utils.models.base_model import BaseModel + + +class DocumentManager(BaseModel): + """Model to store the document details.""" + + document_id = models.UUIDField( + primary_key=True, default=uuid.uuid4, editable=False + ) + + document_name = models.CharField( + db_comment="Field to store the document name", + editable=False, + null=False, + blank=False, + ) + + tool = models.ForeignKey( + CustomTool, + on_delete=models.CASCADE, + related_name="prompt_document_linked_tool", + null=False, + blank=False, + ) + + created_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + related_name="prompt_document_created_by", + null=True, + blank=True, + editable=False, + ) + + modified_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + related_name="prompt_document_modified_by", + null=True, + blank=True, + editable=False, + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["document_name", "tool"], + name="unique_document_name_tool", + ), + ] diff --git a/backend/prompt_studio/prompt_studio_document_manager/prompt_studio_document_helper.py b/backend/prompt_studio/prompt_studio_document_manager/prompt_studio_document_helper.py new file mode 100644 index 000000000..2f101dc73 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/prompt_studio_document_helper.py @@ -0,0 +1,24 @@ +import logging + +from prompt_studio.prompt_studio_core.models import CustomTool + +from .models import DocumentManager + +logger = logging.getLogger(__name__) + + +class PromptStudioDocumentHelper: + + @staticmethod + def create(tool_id: str, document_name: str): + tool: CustomTool = CustomTool.objects.get(pk=tool_id) + document: DocumentManager = DocumentManager.objects.create( + tool=tool, document_name=document_name) + logger.info("Successfully created the record") + return document + + @staticmethod + def delete(document_id: str): + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + document.delete() + logger.info("Successfully deleted the record") diff --git a/backend/prompt_studio/prompt_studio_document_manager/serializers.py b/backend/prompt_studio/prompt_studio_document_manager/serializers.py new file mode 100644 index 000000000..bcad0558a --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/serializers.py @@ -0,0 +1,16 @@ +from backend.serializers import AuditSerializer + +from .constants import PSDMKeys +from .models import DocumentManager + + +class PromptStudioDocumentManagerSerializer(AuditSerializer): + class Meta: + model = DocumentManager + fields = "__all__" + + def to_representation(self, instance): + rep: dict[str, str] = super().to_representation(instance) + required_fields = [PSDMKeys.DOCUMENT_NAME, + PSDMKeys.TOOL, PSDMKeys.DOCUMENT_ID] + return {key: rep[key] for key in required_fields if key in rep} diff --git a/backend/prompt_studio/prompt_studio_document_manager/urls.py b/backend/prompt_studio/prompt_studio_document_manager/urls.py new file mode 100644 index 000000000..208070544 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/urls.py @@ -0,0 +1,32 @@ +from django.urls import path +from rest_framework.urlpatterns import format_suffix_patterns + +from .views import PromptStudioDocumentManagerView + +prompt_studio_documents_list = PromptStudioDocumentManagerView.as_view( + {"get": "list", "post": "create"} +) + +prompt_studio_documents_detail = PromptStudioDocumentManagerView.as_view( + { + "get": "retrieve", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + } +) + +urlpatterns = format_suffix_patterns( + [ + path( + "prompt-document/", + prompt_studio_documents_list, + name="prompt-studio-documents-list", + ), + path( + "prompt-document//", + prompt_studio_documents_detail, + name="tool-studio-documents-detail", + ), + ] +) diff --git a/backend/prompt_studio/prompt_studio_document_manager/views.py b/backend/prompt_studio/prompt_studio_document_manager/views.py new file mode 100644 index 000000000..71596532a --- /dev/null +++ b/backend/prompt_studio/prompt_studio_document_manager/views.py @@ -0,0 +1,31 @@ +from typing import Optional + +from django.db.models import QuerySet +from prompt_studio.prompt_studio_document_manager.serializers import ( + PromptStudioDocumentManagerSerializer, +) +from prompt_studio.prompt_studio_output_manager.constants import ( + PromptStudioOutputManagerKeys, +) +from rest_framework import viewsets +from rest_framework.versioning import URLPathVersioning +from utils.filtering import FilterHelper + +from .models import DocumentManager + + +class PromptStudioDocumentManagerView(viewsets.ModelViewSet): + versioning_class = URLPathVersioning + queryset = DocumentManager.objects.all() + serializer_class = PromptStudioDocumentManagerSerializer + + def get_queryset(self) -> Optional[QuerySet]: + filter_args = FilterHelper.build_filter_args( + self.request, + PromptStudioOutputManagerKeys.TOOL_ID, + ) + if filter_args: + queryset = DocumentManager.objects.filter(**filter_args) + else: + queryset = DocumentManager.objects.all() + return queryset diff --git a/backend/prompt_studio/prompt_studio_index_manager/__init__.py b/backend/prompt_studio/prompt_studio_index_manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/prompt_studio/prompt_studio_index_manager/admin.py b/backend/prompt_studio/prompt_studio_index_manager/admin.py new file mode 100644 index 000000000..188d50ffe --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import IndexManager + +admin.site.register(IndexManager) diff --git a/backend/prompt_studio/prompt_studio_index_manager/apps.py b/backend/prompt_studio/prompt_studio_index_manager/apps.py new file mode 100644 index 000000000..19dc5f79c --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PromptStudioIndexManagerConfig(AppConfig): + name = "prompt_studio.prompt_studio_index_manager" diff --git a/backend/prompt_studio/prompt_studio_index_manager/constants.py b/backend/prompt_studio/prompt_studio_index_manager/constants.py new file mode 100644 index 000000000..2a0588cb6 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/constants.py @@ -0,0 +1,3 @@ +class IndexManagerKeys: + PROFILE_MANAGER = "profile_manager" + DOCUMENT_MANAGER = "document_manager" \ No newline at end of file diff --git a/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py new file mode 100644 index 000000000..3869e2e74 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py @@ -0,0 +1,108 @@ +# Generated by Django 4.2.1 on 2024-03-10 20:48 + +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("prompt_studio_document_manager", "0001_initial"), + ("prompt_profile_manager", "0008_profilemanager_migration"), + ] + + operations = [ + migrations.CreateModel( + name="IndexManager", + fields=[ + ("created_at", models.DateTimeField(auto_now_add=True)), + ("modified_at", models.DateTimeField(auto_now=True)), + ( + "index_manager_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "raw_index_id", + models.CharField( + blank=True, + db_comment="Field to store the raw index id", + editable=False, + null=True, + ), + ), + ( + "summarize_index_id", + models.CharField( + blank=True, + db_comment="Field to store the summarize index id", + editable=False, + null=True, + ), + ), + ( + "index_ids_history", + models.JSONField( + db_comment="List of index ids", default=list), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="prompt_index_manager_created_by", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "document_manager", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="index_manager_linked_document", + to="prompt_studio_document_manager.documentmanager", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="prompt_index_manager_modified_by", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "profile_manager", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="index_manager_linked_raw_llm_profile", + to="prompt_profile_manager.profilemanager", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="indexmanager", + constraint=models.UniqueConstraint( + fields=("document_manager", "profile_manager"), + name="unique_document_manager_profile_manager", + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_index_manager/migrations/__init__.py b/backend/prompt_studio/prompt_studio_index_manager/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/prompt_studio/prompt_studio_index_manager/models.py b/backend/prompt_studio/prompt_studio_index_manager/models.py new file mode 100644 index 000000000..220688cb9 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/models.py @@ -0,0 +1,80 @@ +import uuid + +from account.models import User +from django.db import models +from prompt_studio.prompt_profile_manager.models import ProfileManager +from prompt_studio.prompt_studio_document_manager.models import DocumentManager +from utils.models.base_model import BaseModel + + +class IndexManager(BaseModel): + """Model to store the index details.""" + + index_manager_id = models.UUIDField( + primary_key=True, default=uuid.uuid4, editable=False + ) + + document_manager = models.ForeignKey( + DocumentManager, + on_delete=models.CASCADE, + related_name="index_manager_linked_document", + editable=False, + null=False, + blank=False, + ) + + profile_manager = models.ForeignKey( + ProfileManager, + on_delete=models.SET_NULL, + related_name="index_manager_linked_raw_llm_profile", + editable=False, + null=True, + blank=True, + ) + + raw_index_id = models.CharField( + db_comment="Field to store the raw index id", + editable=False, + null=True, + blank=True, + ) + + summarize_index_id = models.CharField( + db_comment="Field to store the summarize index id", + editable=False, + null=True, + blank=True, + ) + + index_ids_history = models.JSONField( + db_comment="List of index ids", + default=list, + null=False, + blank=False, + ) + + created_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + related_name="prompt_index_manager_created_by", + null=True, + blank=True, + editable=False, + ) + + modified_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + related_name="prompt_index_manager_modified_by", + null=True, + blank=True, + editable=False, + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["document_manager", "profile_manager"], + name="unique_document_manager_profile_manager", + ), + ] diff --git a/backend/prompt_studio/prompt_studio_index_manager/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager/prompt_studio_index_helper.py new file mode 100644 index 000000000..c044f475d --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/prompt_studio_index_helper.py @@ -0,0 +1,57 @@ +import json +import logging + +from prompt_studio.prompt_profile_manager.models import ProfileManager +from prompt_studio.prompt_studio_document_manager.models import DocumentManager + +from .models import IndexManager + +logger = logging.getLogger(__name__) + + +class PromptStudioIndexHelper: + + @staticmethod + def handle_index_manager( + document_id: str, + is_summary: bool, + profile_manager: ProfileManager, + doc_id: str + ): + document: DocumentManager = DocumentManager.objects.get(pk=document_id) + + index_id = "raw_index_id" + if is_summary: + index_id = "summarize_index_id" + + try: + index_manager: IndexManager = IndexManager.objects.get(**{ + "document_manager": document, + "profile_manager": profile_manager, + }) + except Exception: + index_manager = None + + args: dict = { + f'{index_id}': doc_id, + } + + index_ids_list = [] + if index_manager: + index_ids = index_manager.index_ids_history + index_ids_list = json.loads(index_ids) if index_ids else [] + if doc_id not in index_ids: + index_ids_list.append(doc_id) + else: + index_ids_list.append(doc_id) + + args["index_ids_history"] = json.dumps(index_ids_list) + + if index_manager: + result: IndexManager = IndexManager.objects.filter( + index_manager_id=index_manager.index_manager_id).update(**args) + else: + args["document_manager"] = document + args["profile_manager"] = profile_manager + result: IndexManager = IndexManager.objects.create(**args) + return result diff --git a/backend/prompt_studio/prompt_studio_index_manager/serializers.py b/backend/prompt_studio/prompt_studio_index_manager/serializers.py new file mode 100644 index 000000000..ae33a186e --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/serializers.py @@ -0,0 +1,9 @@ +from backend.serializers import AuditSerializer + +from .models import IndexManager + + +class IndexManagerSerializer(AuditSerializer): + class Meta: + model = IndexManager + fields = "__all__" diff --git a/backend/prompt_studio/prompt_studio_index_manager/urls.py b/backend/prompt_studio/prompt_studio_index_manager/urls.py new file mode 100644 index 000000000..63858612b --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/urls.py @@ -0,0 +1,32 @@ +from django.urls import path +from rest_framework.urlpatterns import format_suffix_patterns + +from .views import IndexManagerView + +prompt_studio_index_list = IndexManagerView.as_view( + {"get": "list", "post": "create"} +) + +prompt_studio_index_detail = IndexManagerView.as_view( + { + "get": "retrieve", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + } +) + +urlpatterns = format_suffix_patterns( + [ + path( + "document-index/", + prompt_studio_index_list, + name="prompt-studio-documents-list", + ), + path( + "document-index//", + prompt_studio_index_detail, + name="tool-studio-documents-detail", + ), + ] +) diff --git a/backend/prompt_studio/prompt_studio_index_manager/views.py b/backend/prompt_studio/prompt_studio_index_manager/views.py new file mode 100644 index 000000000..9a870f029 --- /dev/null +++ b/backend/prompt_studio/prompt_studio_index_manager/views.py @@ -0,0 +1,30 @@ +from typing import Optional + +from django.db.models import QuerySet +from prompt_studio.prompt_studio_index_manager.constants import IndexManagerKeys +from prompt_studio.prompt_studio_index_manager.serializers import ( + IndexManagerSerializer, +) +from rest_framework import viewsets +from rest_framework.versioning import URLPathVersioning +from utils.filtering import FilterHelper + +from .models import IndexManager + + +class IndexManagerView(viewsets.ModelViewSet): + versioning_class = URLPathVersioning + queryset = IndexManager.objects.all() + serializer_class = IndexManagerSerializer + + def get_queryset(self) -> Optional[QuerySet]: + filter_args = FilterHelper.build_filter_args( + self.request, + IndexManagerKeys.PROFILE_MANAGER, + IndexManagerKeys.DOCUMENT_MANAGER, + ) + if filter_args: + queryset = IndexManager.objects.filter(**filter_args) + else: + queryset = IndexManager.objects.all() + return queryset diff --git a/backend/prompt_studio/prompt_studio_output_manager/constants.py b/backend/prompt_studio/prompt_studio_output_manager/constants.py index f2cbd9b94..7308aab97 100644 --- a/backend/prompt_studio/prompt_studio_output_manager/constants.py +++ b/backend/prompt_studio/prompt_studio_output_manager/constants.py @@ -2,4 +2,4 @@ class PromptStudioOutputManagerKeys: TOOL_ID = "tool_id" PROMPT_ID = "prompt_id" PROFILE_MANAGER = "profile_manager" - DOC_NAME = "doc_name" + DOCUMENT_MANAGER = "document_manager" diff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0008_remove_promptstudiooutputmanager_doc_name_and_more.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0008_remove_promptstudiooutputmanager_doc_name_and_more.py new file mode 100644 index 000000000..3231c38cd --- /dev/null +++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0008_remove_promptstudiooutputmanager_doc_name_and_more.py @@ -0,0 +1,65 @@ +# Generated by Django 4.2.1 on 2024-03-10 20:48 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("prompt_studio_document_manager", "0001_initial"), + ("prompt_studio_core", "0007_remove_customtool_default_profile_and_more"), + ("prompt_profile_manager", "0008_profilemanager_migration"), + ("prompt_studio", "0006_alter_toolstudioprompt_prompt_key_and_more"), + ("prompt_studio_output_manager", "0007_promptstudiooutputmanager_eval_metrics"), + ] + + operations = [ + migrations.RemoveField( + model_name="promptstudiooutputmanager", + name="doc_name", + ), + migrations.AddField( + model_name="promptstudiooutputmanager", + name="document_manager", + field=models.ForeignKey( + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_output_linked_document_manager", + to="prompt_studio_document_manager.documentmanager", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="promptstudiooutputmanager", + name="profile_manager", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_output_linked_prompt", + to="prompt_profile_manager.profilemanager", + ), + ), + migrations.AlterField( + model_name="promptstudiooutputmanager", + name="prompt_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_output_linked_prompt", + to="prompt_studio.toolstudioprompt", + ), + ), + migrations.AlterField( + model_name="promptstudiooutputmanager", + name="tool_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_ouput_linked_tool", + to="prompt_studio_core.customtool", + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_output_manager/models.py b/backend/prompt_studio/prompt_studio_output_manager/models.py index 1e12d7cd2..6b2c3eee3 100644 --- a/backend/prompt_studio/prompt_studio_output_manager/models.py +++ b/backend/prompt_studio/prompt_studio_output_manager/models.py @@ -4,6 +4,7 @@ from django.db import models from prompt_studio.prompt_profile_manager.models import ProfileManager from prompt_studio.prompt_studio.models import ToolStudioPrompt +from prompt_studio.prompt_studio_document_manager.models import DocumentManager from prompt_studio.prompt_studio_core.models import CustomTool from utils.models.base_model import BaseModel @@ -17,13 +18,13 @@ class PromptStudioOutputManager(BaseModel): prompt_output_id = models.UUIDField( primary_key=True, default=uuid.uuid4, editable=False ) - output = models.CharField( db_comment="Field to store output", editable=True, null=True, blank=True ) - - doc_name = models.CharField( - db_comment="Field to store the document name", + document_manager = models.ForeignKey( + DocumentManager, + on_delete=models.CASCADE, + related_name="prompt_output_linked_document_manager", editable=True, ) eval_metrics = models.JSONField( @@ -35,21 +36,21 @@ class PromptStudioOutputManager(BaseModel): ) tool_id = models.ForeignKey( CustomTool, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, related_name="prompt_ouput_linked_tool", null=True, blank=True, ) prompt_id = models.ForeignKey( ToolStudioPrompt, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, related_name="prompt_output_linked_prompt", null=True, blank=True, ) profile_manager = models.ForeignKey( ProfileManager, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, related_name="prompt_output_linked_prompt", null=True, blank=True, diff --git a/backend/prompt_studio/prompt_studio_output_manager/views.py b/backend/prompt_studio/prompt_studio_output_manager/views.py index f9f743a3c..f87a31ebe 100644 --- a/backend/prompt_studio/prompt_studio_output_manager/views.py +++ b/backend/prompt_studio/prompt_studio_output_manager/views.py @@ -28,7 +28,7 @@ def get_queryset(self) -> Optional[QuerySet]: PromptStudioOutputManagerKeys.TOOL_ID, PromptStudioOutputManagerKeys.PROMPT_ID, PromptStudioOutputManagerKeys.PROFILE_MANAGER, - PromptStudioOutputManagerKeys.DOC_NAME, + PromptStudioOutputManagerKeys.DOCUMENT_MANAGER, ) if filter_args: queryset = PromptStudioOutputManager.objects.filter(**filter_args) diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx index 9cc026c45..181993bcf 100644 --- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx +++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx @@ -21,7 +21,7 @@ const outputTypes = { yaml: "YAML", }; -function CombinedOutput({ doc, setFilledFields }) { +function CombinedOutput({ docId, setFilledFields }) { const [combinedOutput, setCombinedOutput] = useState({}); const [yamlData, setYamlData] = useState(null); const [selectedOutputType, setSelectedOutputType] = useState( @@ -34,7 +34,7 @@ function CombinedOutput({ doc, setFilledFields }) { const axiosPrivate = useAxiosPrivate(); useEffect(() => { - if (!doc) { + if (!docId) { return; } @@ -91,7 +91,7 @@ function CombinedOutput({ doc, setFilledFields }) { .finally(() => { setIsOutputLoading(false); }); - }, [doc]); + }, [docId]); useEffect(() => { Prism.highlightAll(); @@ -104,7 +104,7 @@ function CombinedOutput({ doc, setFilledFields }) { const handleOutputApiRequest = async () => { const requestOptions = { method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&doc_name=${doc}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&document_manager=${docId}`, headers: { "X-CSRFToken": sessionDetails?.csrfToken, }, @@ -151,7 +151,7 @@ function CombinedOutput({ doc, setFilledFields }) { } CombinedOutput.propTypes = { - doc: PropTypes.string, + docId: PropTypes.string, setFilledFields: PropTypes.func, }; diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.css b/frontend/src/components/custom-tools/document-manager/DocumentManager.css index 97a507856..f28432a7f 100644 --- a/frontend/src/components/custom-tools/document-manager/DocumentManager.css +++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.css @@ -4,7 +4,7 @@ display: flex; flex-direction: column; height: 100%; - overflow-y: hidden; + overflow: hidden; } .doc-manager-header { @@ -27,6 +27,7 @@ .text-viewer-layout { padding: 0px 12px; + flex: 1; overflow-y: auto; } @@ -36,5 +37,5 @@ .doc-main-title{ color: darkgray; - font-weight: 600; + font-size: 12px; } diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx index 850e3dc91..cebc662d5 100644 --- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx +++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx @@ -12,12 +12,9 @@ import { ManageDocsModal } from "../manage-docs-modal/ManageDocsModal"; import { useEffect, useState } from "react"; import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate"; import { useSessionStore } from "../../../store/session-store"; -import { - base64toBlob, - removeFileExtension, -} from "../../../helpers/GetStaticData"; +import { base64toBlob } from "../../../helpers/GetStaticData"; import { DocumentViewer } from "../document-viewer/DocumentViewer"; -import { TextViewer } from "../text-viewer/TextViewer"; +import { TextViewerPre } from "../text-viewer-pre/TextViewerPre"; const items = [ { @@ -31,7 +28,7 @@ const items = [ ]; const viewTypes = { - pdf: "PDF", + original: "ORIGINAL", extract: "EXTRACT", }; @@ -65,27 +62,15 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { const axiosPrivate = useAxiosPrivate(); useEffect(() => { - const fileNameTxt = removeFileExtension(selectedDoc); - const files = [ - { - fileName: selectedDoc, - viewType: viewTypes.pdf, - }, - { - fileName: `extract/${fileNameTxt}.txt`, - viewType: viewTypes.extract, - }, - ]; - setFileUrl(""); setExtractTxt(""); - files.forEach((item) => { - handleFetchContent(item); + Object.keys(viewTypes).forEach((item) => { + handleFetchContent(viewTypes[item]); }); }, [selectedDoc]); - const handleFetchContent = (fileDetails) => { - if (!selectedDoc) { + const handleFetchContent = (viewType) => { + if (!selectedDoc?.document_id) { setFileUrl(""); setExtractTxt(""); return; @@ -93,32 +78,32 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { const requestOptions = { method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?file_name=${fileDetails?.fileName}&tool_id=${details?.tool_id}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?document_id=${selectedDoc?.document_id}&view_type=${viewType}&tool_id=${details?.tool_id}`, }; - handleLoadingStateUpdate(fileDetails?.viewType, true); + handleLoadingStateUpdate(viewType, true); axiosPrivate(requestOptions) .then((res) => { const data = res?.data?.data; - if (fileDetails?.viewType === viewTypes.pdf) { + if (viewType === viewTypes.original) { const base64String = data || ""; const blob = base64toBlob(base64String); setFileUrl(URL.createObjectURL(blob)); return; } - if (fileDetails?.viewType === viewTypes?.extract) { + if (viewType === viewTypes?.extract) { setExtractTxt(data); } }) .catch((err) => {}) .finally(() => { - handleLoadingStateUpdate(fileDetails?.viewType, false); + handleLoadingStateUpdate(viewType, false); }); }; const handleLoadingStateUpdate = (viewType, value) => { - if (viewType === viewTypes.pdf) { + if (viewType === viewTypes.original) { setIsDocLoading(value); } @@ -132,7 +117,9 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { }; useEffect(() => { - const index = [...listOfDocs].findIndex((item) => item === selectedDoc); + const index = [...listOfDocs].findIndex( + (item) => item?.document_id === selectedDoc?.document_id + ); setPage(index + 1); }, [selectedDoc, listOfDocs]); @@ -157,7 +144,7 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { const updatePageAndDoc = (newPage) => { setPage(newPage); const newSelectedDoc = listOfDocs[newPage - 1]; - handleDocChange(newSelectedDoc); + handleDocChange(newSelectedDoc?.document_id); }; return ( @@ -174,7 +161,7 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
{selectedDoc ? ( - {selectedDoc} + {selectedDoc?.document_name} ) : null}
@@ -216,7 +203,7 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { {activeKey === "1" && ( 0} setOpenManageDocsModal={setOpenManageDocsModal} @@ -226,12 +213,12 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { )} {activeKey === "2" && ( 0} setOpenManageDocsModal={setOpenManageDocsModal} > - + )} {SummarizeView && activeKey === 3 && ( diff --git a/frontend/src/components/custom-tools/editable-text/EditableText.jsx b/frontend/src/components/custom-tools/editable-text/EditableText.jsx index 466b40115..c53ef82ea 100644 --- a/frontend/src/components/custom-tools/editable-text/EditableText.jsx +++ b/frontend/src/components/custom-tools/editable-text/EditableText.jsx @@ -20,7 +20,8 @@ function EditableText({ const [triggerHandleChange, setTriggerHandleChange] = useState(false); const [isHovered, setIsHovered] = useState(false); const divRef = useRef(null); - const { disableLlmOrDocChange } = useCustomToolStore(); + const { disableLlmOrDocChange, indexDocs, selectedDoc } = + useCustomToolStore(); useEffect(() => { setText(defaultText); @@ -104,7 +105,10 @@ function EditableText({ onMouseOut={() => setIsHovered(false)} onBlur={handleBlur} onClick={() => setIsEditing(true)} - disabled={disableLlmOrDocChange.includes(promptId)} + disabled={ + disableLlmOrDocChange.includes(promptId) || + indexDocs.includes(selectedDoc?.document_id) + } /> ); } diff --git a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.css b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.css index 602171c75..d96eacfc1 100644 --- a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.css +++ b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.css @@ -42,4 +42,6 @@ .list-of-tools-body { flex: 1; overflow-y: auto; + display: flex; + justify-content: center; } \ No newline at end of file diff --git a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx index b41cac96f..7f2779c79 100644 --- a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx +++ b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx @@ -1,12 +1,5 @@ -import { - AppstoreOutlined, - BarsOutlined, - PlusOutlined, -} from "@ant-design/icons"; -import { Input, Segmented, Typography } from "antd"; -import debounce from "lodash/debounce"; -import isEmpty from "lodash/isEmpty"; -import { useCallback, useEffect, useState } from "react"; +import { PlusOutlined } from "@ant-design/icons"; +import { useEffect, useState } from "react"; import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate"; import { useAlertStore } from "../../../store/alert-store"; @@ -15,22 +8,10 @@ import { CustomButton } from "../../widgets/custom-button/CustomButton"; import { AddCustomToolFormModal } from "../add-custom-tool-form-modal/AddCustomToolFormModal"; import { ViewTools } from "../view-tools/ViewTools"; import { handleException } from "../../../helpers/GetStaticData"; +import { ToolNavBar } from "../../navigations/tool-nav-bar/ToolNavBar"; import "./ListOfTools.css"; -const { Search } = Input; - function ListOfTools() { - const VIEW_OPTIONS = [ - { - value: "grid", - icon: , - }, - { - value: "list", - icon: , - }, - ]; - const [viewType, setViewType] = useState(VIEW_OPTIONS[0].value); const [isListLoading, setIsListLoading] = useState(false); const [openAddTool, setOpenAddTool] = useState(false); const [editItem, setEditItem] = useState(null); @@ -39,14 +20,17 @@ function ListOfTools() { const axiosPrivate = useAxiosPrivate(); const [listOfTools, setListOfTools] = useState([]); - const [filteredListOfTools, setFilteredListOfTools] = useState([]); - const [search, setSearch] = useState(""); + const [filteredListOfTools, setFilteredListOfTools] = useState(listOfTools); const [isEdit, setIsEdit] = useState(false); useEffect(() => { getListOfTools(); }, []); + useEffect(() => { + setFilteredListOfTools(listOfTools); + }, [listOfTools]); + const getListOfTools = () => { const requestOptions = { method: "GET", @@ -61,6 +45,7 @@ function ListOfTools() { .then((res) => { const data = res?.data; setListOfTools(data); + setFilteredListOfTools(data); }) .catch((err) => { setAlertDetails( @@ -118,9 +103,10 @@ function ListOfTools() { setListOfTools(tools); }; - const handleEdit = (event, id) => { - event.domEvent.stopPropagation(); - const editToolData = [...listOfTools].find((item) => item?.tool_id === id); + const handleEdit = (_event, tool) => { + const editToolData = [...listOfTools].find( + (item) => item?.tool_id === tool.tool_id + ); if (!editToolData) { return; } @@ -129,10 +115,10 @@ function ListOfTools() { setOpenAddTool(true); }; - const handleDelete = (event, id) => { + const handleDelete = (_event, tool) => { const requestOptions = { method: "DELETE", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/${id}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/${tool.tool_id}`, headers: { "X-CSRFToken": sessionDetails?.csrfToken, }, @@ -140,7 +126,9 @@ function ListOfTools() { axiosPrivate(requestOptions) .then(() => { - const tools = [...listOfTools].filter((tool) => tool?.tool_id !== id); + const tools = [...listOfTools].filter( + (filterToll) => filterToll?.tool_id !== tool.tool_id + ); setListOfTools(tools); }) .catch((err) => { @@ -153,28 +141,17 @@ function ListOfTools() { }); }; - const handleViewChange = (type) => { - setViewType(type); - }; - - useEffect(() => { + const onSearch = (search, setSearch) => { if (search?.length === 0) { - setFilteredListOfTools(listOfTools); + setSearch(listOfTools); } const filteredList = [...listOfTools].filter((tool) => { const name = tool.tool_name?.toUpperCase(); const searchUpperCase = search.toUpperCase(); return name.includes(searchUpperCase); }); - setFilteredListOfTools(filteredList); - }, [search, listOfTools]); - - const onSearchDebounce = useCallback( - debounce(({ target: { value } }) => { - setSearch(value); - }, 600), - [] - ); + setSearch(filteredList); + }; const showAddTool = () => { setEditItem(null); @@ -182,48 +159,43 @@ function ListOfTools() { setOpenAddTool(true); }; + const CustomButtons = () => { + return ( + } + onClick={showAddTool} + > + New Tool + + ); + }; + return ( <> +
-
-
- - Prompt Studio - -
- - - } - onClick={showAddTool} - > - New Tool - -
-
-
-
- -
+
+
diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx index d08de85b2..077b73f0c 100644 --- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx +++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx @@ -1,5 +1,8 @@ import { + CheckCircleFilled, + CloseCircleFilled, DeleteOutlined, + InfoCircleFilled, PlusOutlined, ReloadOutlined, } from "@ant-design/icons"; @@ -8,6 +11,8 @@ import { Divider, Modal, Radio, + Select, + Space, Table, Tooltip, Typography, @@ -25,6 +30,12 @@ import { ConfirmModal } from "../../widgets/confirm-modal/ConfirmModal"; import { EmptyState } from "../../widgets/empty-state/EmptyState"; import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper"; import "./ManageDocsModal.css"; +import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader"; + +const indexTypes = { + raw: "RAW", + summarize: "Summarize", +}; function ManageDocsModal({ open, @@ -35,24 +46,191 @@ function ManageDocsModal({ }) { const [isUploading, setIsUploading] = useState(false); const [rows, setRows] = useState([]); + const [rawLlmProfile, setRawLlmProfile] = useState(""); + const [isRawDataLoading, setIsRawDataLoading] = useState(false); + const [rawIndexStatus, setRawIndexStatus] = useState([]); + const [summarizeLlmProfile, setSummarizeLlmProfile] = useState(""); + const [isSummarizeDataLoading, setIsSummarizeDataLoading] = useState(false); + const [summarizeIndexStatus, setSummarizeIndexStatus] = useState([]); + const [llmItems, setLlmItems] = useState([]); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const { selectedDoc, listOfDocs, + llmProfiles, updateCustomTool, details, defaultLlmProfile, disableLlmOrDocChange, + indexDocs, } = useCustomToolStore(); const axiosPrivate = useAxiosPrivate(); + const successIndex = ( + + + + {" "} + Indexed + + ); + + const failedIndex = ( + + + + {" "} + Not Indexed + + ); + + const infoIndex = ( + + + + {" "} + LLM Profile not selected + + ); + + useEffect(() => { + setRawLlmProfile(defaultLlmProfile); + setSummarizeLlmProfile(details?.summarize_llm_profile); + }, [llmItems]); + + useEffect(() => { + getLlmProfilesDropdown(); + }, [llmProfiles]); + + useEffect(() => { + handleGetIndexStatus(rawLlmProfile, indexTypes.raw); + }, [rawLlmProfile, indexDocs]); + + useEffect(() => { + handleGetIndexStatus(summarizeLlmProfile, indexTypes.summarize); + }, [summarizeLlmProfile, indexDocs]); + + const handleLoading = (indexType, value) => { + if (indexType === indexTypes.raw) { + setIsRawDataLoading(value); + } + + if (indexType === indexTypes.summarize) { + setIsSummarizeDataLoading(value); + } + }; + + const handleIndexStatus = (indexType, data) => { + if (indexType === indexTypes.raw) { + setRawIndexStatus(data); + } + + if (indexType === indexTypes.summarize) { + setSummarizeIndexStatus(data); + } + }; + + const handleIsIndexed = (indexType, data) => { + let isIndexed = false; + if (indexType === indexTypes.raw) { + isIndexed = !!data?.raw_index_id; + } + + if (indexType === indexTypes.summarize) { + isIndexed = !!data?.summarize_index_id; + } + + return isIndexed; + }; + + const handleGetIndexStatus = (llmProfileId, indexType) => { + if (!llmProfileId) { + return; + } + + const requestOptions = { + method: "GET", + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/document-index?profile_manager=${llmProfileId}`, + }; + + handleLoading(indexType, true); + axiosPrivate(requestOptions) + .then((res) => { + const data = res?.data; + const indexStatus = data.map((item) => { + return { + docId: item?.document_manager, + isIndexed: handleIsIndexed(indexType, item), + }; + }); + + handleIndexStatus(indexType, indexStatus); + }) + .catch((err) => { + setAlertDetails(handleException(err, "Failed to get index status")); + }) + .finally(() => { + handleLoading(indexType, false); + }); + }; + + const getLlmProfilesDropdown = () => { + const items = [...llmProfiles].map((item) => { + return { + value: item?.profile_id, + label: item?.profile_name, + }; + }); + setLlmItems(items); + }; + const columns = [ { title: "Document", dataIndex: "document", key: "document", }, + { + title: ( + + Raw + setSummarizeLlmProfile(value)} + /> + {isSummarizeDataLoading && } + + ), + dataIndex: "summarizeIndex", + key: "summarizeIndex", + width: 250, + }, { title: "", dataIndex: "reindex", @@ -73,31 +251,58 @@ function ManageDocsModal({ }, ]; + const getIndexStatusMessage = (docId, indexType) => { + let instance = null; + if (indexType === indexTypes.raw) { + if (!rawLlmProfile) { + return infoIndex; + } + instance = rawIndexStatus.find((item) => item?.docId === docId); + } else { + if (!summarizeLlmProfile) { + return infoIndex; + } + instance = summarizeIndexStatus.find((item) => item?.docId === docId); + } + + return instance?.isIndexed ? successIndex : failedIndex; + }; + useEffect(() => { - const newRows = listOfDocs.map((doc) => { + const newRows = listOfDocs.map((item) => { return { - key: doc, - document: doc || "", - reindex: ( - - - + key: item?.document_id, + document: item?.document_name || "", + rawIndex: getIndexStatusMessage(item?.document_id, indexTypes.raw), + summarizeIndex: getIndexStatusMessage( + item?.document_id, + indexTypes.summarize + ), + reindex: indexDocs.includes(item?.document_id) ? ( + + ) : ( + @@ -105,15 +310,25 @@ function ManageDocsModal({ ), select: ( handleDocChange(doc)} - disabled={disableLlmOrDocChange?.length > 0} + checked={selectedDoc?.document_id === item?.document_id} + onClick={() => handleDocChange(item?.document_id)} + disabled={ + disableLlmOrDocChange?.length > 0 || + indexDocs.includes(item?.document_id) + } /> ), }; }); setRows(newRows); - }, [listOfDocs, selectedDoc, disableLlmOrDocChange]); + }, [ + listOfDocs, + selectedDoc, + disableLlmOrDocChange, + rawIndexStatus, + summarizeIndexStatus, + indexDocs, + ]); const beforeUpload = (file) => { return new Promise((resolve, reject) => { @@ -121,7 +336,9 @@ function ManageDocsModal({ reader.readAsDataURL(file); reader.onload = () => { const fileName = file.name; - const fileAlreadyExists = [...listOfDocs].includes(fileName); + const fileAlreadyExists = [...listOfDocs].find( + (item) => item?.document_name === fileName + ); if (!fileAlreadyExists) { resolve(file); } else { @@ -147,17 +364,15 @@ function ManageDocsModal({ content: "File uploaded successfully", }); - const docName = info?.file?.name; + const data = info.file.response?.data; + const doc = data?.length > 0 ? data[0] : {}; const newListOfDocs = [...listOfDocs]; - newListOfDocs.push(docName); - setOpen(false); - await generateIndex(info?.file?.name); + newListOfDocs.push(doc); const body = { - selectedDoc: docName, listOfDocs: newListOfDocs, }; updateCustomTool(body); - handleUpdateTool({ output: docName }); + handleUpdateTool({ output: doc?.document_id }); } else if (info.file.status === "error") { setIsUploading(false); setAlertDetails({ @@ -167,20 +382,20 @@ function ManageDocsModal({ } }; - const handleDelete = (docName) => { + const handleDelete = (docId) => { const requestOptions = { method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/file/delete?file_name=${docName}&tool_id=${details?.tool_id}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/file/delete?document_id=${docId}&tool_id=${details?.tool_id}`, }; axiosPrivate(requestOptions) .then(() => { const newListOfDocs = [...listOfDocs].filter( - (item) => item !== docName + (item) => item?.document_id !== docId ); updateCustomTool({ listOfDocs: newListOfDocs }); - if (docName === selectedDoc) { + if (docId === selectedDoc?.document_id) { updateCustomTool({ selectedDoc: "" }); handleUpdateTool({ output: "" }); } @@ -198,6 +413,7 @@ function ManageDocsModal({ centered footer={null} maskClosable={false} + width={1000} >
diff --git a/frontend/src/components/custom-tools/output-analyzer-card/OutputAnalyzerCard.jsx b/frontend/src/components/custom-tools/output-analyzer-card/OutputAnalyzerCard.jsx index a0b60e905..de61075d7 100644 --- a/frontend/src/components/custom-tools/output-analyzer-card/OutputAnalyzerCard.jsx +++ b/frontend/src/components/custom-tools/output-analyzer-card/OutputAnalyzerCard.jsx @@ -27,7 +27,7 @@ function OutputAnalyzerCard({ doc, totalFields }) { const requestOptions = { method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?file_name=${doc}&tool_id=${details?.tool_id}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?document_id=${doc?.document_id}&tool_id=${details?.tool_id}`, }; setIsDocLoading(true); @@ -49,7 +49,7 @@ function OutputAnalyzerCard({ doc, totalFields }) {
- {doc} + {doc?.document_name}
@@ -92,7 +92,10 @@ function OutputAnalyzerCard({ doc, totalFields }) {
- +
@@ -102,7 +105,7 @@ function OutputAnalyzerCard({ doc, totalFields }) { } OutputAnalyzerCard.propTypes = { - doc: PropTypes.string.isRequired, + doc: PropTypes.object.isRequired, totalFields: PropTypes.number.isRequired, }; diff --git a/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx index 7b8b3d392..41c7aa1bd 100644 --- a/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx +++ b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx @@ -25,7 +25,7 @@ function OutputAnalyzerList() {
{listOfDocs.map((doc) => { return ( -
+
); diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx index 8bc293ddf..e6f29c6ca 100644 --- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx +++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx @@ -69,12 +69,14 @@ function OutputForDocModal({ const handleRowsGeneration = (data) => { const rowsData = []; [...listOfDocs].forEach((item) => { - const output = data.find((outputValue) => outputValue?.doc_name === item); + const output = data.find( + (outputValue) => outputValue?.document_manager === item?.document_id + ); const isSuccess = output?.output?.length > 0; const result = { key: item, - document: item, + document: item?.document_name, value: ( diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx index e3fc2b8ab..633de4d87 100644 --- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx +++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx @@ -103,6 +103,7 @@ function PromptCard({ updateCustomTool, details, disableLlmOrDocChange, + indexDocs, } = useCustomToolStore(); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); @@ -280,7 +281,7 @@ function PromptCard({ method = "PATCH"; url += `${result?.promptOutputId}/`; } - handleRunApiRequest(selectedDoc) + handleRunApiRequest(selectedDoc?.document_id) .then((res) => { const data = res?.data; const value = data[promptDetails?.prompt_key]; @@ -294,12 +295,21 @@ function PromptCard({ promptDetails?.prompt_key, data ); - handleUpdateOutput(value, selectedDoc, evalMetrics, method, url); + handleUpdateOutput( + value, + selectedDoc?.document_id, + evalMetrics, + method, + url + ); }) .catch((err) => { - handleUpdateOutput(null, selectedDoc, [], method, url); + handleUpdateOutput(null, selectedDoc?.document_id, [], method, url); setAlertDetails( - handleException(err, `Failed to generate output for ${selectedDoc}`) + handleException( + err, + `Failed to generate output for ${selectedDoc?.document_id}` + ) ); }) .finally(() => { @@ -312,7 +322,7 @@ function PromptCard({ // Get the coverage for all the documents except the one that's currently selected const handleCoverage = () => { const listOfDocsToProcess = [...listOfDocs].filter( - (item) => item !== selectedDoc + (item) => item?.document_id !== selectedDoc?.document_id ); if (listOfDocsToProcess?.length === 0) { @@ -323,12 +333,14 @@ function PromptCard({ listOfDocsToProcess.forEach((item) => { let method = "POST"; let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`; - const outputId = outputIds.find((output) => output?.docName === item); + const outputId = outputIds.find( + (output) => output?.docId === item?.document_id + ); if (outputId?.promptOutputId?.length) { method = "PATCH"; url += `${outputId?.promptOutputId}/`; } - handleRunApiRequest(item) + handleRunApiRequest(item?.document_id) .then((res) => { const data = res?.data; const outputValue = data[promptDetails?.prompt_key]; @@ -342,12 +354,21 @@ function PromptCard({ promptDetails?.prompt_key, data ); - handleUpdateOutput(outputValue, item, evalMetrics, method, url); + handleUpdateOutput( + outputValue, + item?.document_id, + evalMetrics, + method, + url + ); }) .catch((err) => { - handleUpdateOutput(null, item, [], method, url); + handleUpdateOutput(null, item?.document_id, [], method, url); setAlertDetails( - handleException(err, `Failed to generate output for ${item}`) + handleException( + err, + `Failed to generate output for ${item?.document_id}` + ) ); }) .finally(() => { @@ -356,11 +377,11 @@ function PromptCard({ }); }; - const handleRunApiRequest = async (doc) => { + const handleRunApiRequest = async (docId) => { const promptId = promptDetails?.prompt_id; const body = { - file_name: doc, + document_id: docId, id: promptId, tool_id: details?.tool_id, }; @@ -382,13 +403,7 @@ function PromptCard({ }); }; - const handleUpdateOutput = ( - outputValue, - docName, - evalMetrics, - method, - url - ) => { + const handleUpdateOutput = (outputValue, docId, evalMetrics, method, url) => { let output = outputValue; if (output !== null && typeof output !== "string") { output = JSON.stringify(output); @@ -398,7 +413,7 @@ function PromptCard({ tool_id: details?.tool_id, prompt_id: promptDetails?.prompt_id, profile_manager: promptDetails?.profile_manager, - doc_name: docName, + document_manager: docId, eval_metrics: evalMetrics, }; @@ -416,7 +431,7 @@ function PromptCard({ .then((res) => { const data = res?.data; const promptOutputId = data?.prompt_output_id || null; - if (docName === selectedDoc) { + if (docId === selectedDoc?.document_id) { setResult({ promptOutputId: promptOutputId, output: data?.output, @@ -429,7 +444,7 @@ function PromptCard({ ); if (!isOutputIdAvailable) { const listOfOutputIds = [...outputIds]; - listOfOutputIds.push({ promptOutputId, docName }); + listOfOutputIds.push({ promptOutputId, docId }); setOutputIds(listOfOutputIds); } }) @@ -492,7 +507,7 @@ function PromptCard({ let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${selectedLlmProfileId}`; if (isOutput) { - url += `&doc_name=${selectedDoc}`; + url += `&document_manager=${selectedDoc?.document_id}`; } const requestOptions = { method: "GET", @@ -519,7 +534,7 @@ function PromptCard({ const ids = []; data.forEach((item) => { const isOutputAdded = ids.findIndex( - (output) => output?.docName === item?.doc_name + (output) => output?.docId === item?.document_manager ); if (isOutputAdded > -1) { @@ -528,11 +543,13 @@ function PromptCard({ if ( item?.output !== undefined && - [...listOfDocs].includes(item?.doc_name) + [...listOfDocs].find( + (doc) => doc?.document_id === item?.document_manager + ) ) { ids.push({ promptOutputId: item?.prompt_output_id, - docName: item?.doc_name, + docId: item?.document_manager, }); } }); @@ -566,9 +583,10 @@ function PromptCard({ defaultValue={promptDetails?.assert_prompt} name="assert_prompt" onChange={onSearchDebounce} - disabled={disableLlmOrDocChange.includes( - promptDetails?.prompt_id - )} + disabled={ + disableLlmOrDocChange.includes(promptDetails?.prompt_id) || + indexDocs.includes(selectedDoc?.document_id) + } /> @@ -581,9 +599,10 @@ function PromptCard({ defaultValue={promptDetails?.assertion_failure_prompt} name="assertion_failure_prompt" onChange={onSearchDebounce} - disabled={disableLlmOrDocChange.includes( - promptDetails?.prompt_id - )} + disabled={ + disableLlmOrDocChange.includes(promptDetails?.prompt_id) || + indexDocs.includes(selectedDoc?.document_id) + } /> @@ -672,9 +691,11 @@ function PromptCard({ type="text" className="display-flex-align-center" onClick={enableEdit} - disabled={disableLlmOrDocChange.includes( - promptDetails?.prompt_id - )} + disabled={ + disableLlmOrDocChange.includes( + promptDetails?.prompt_id + ) || indexDocs.includes(selectedDoc?.document_id) + } > @@ -699,7 +720,10 @@ function PromptCard({ (updateStatus?.promptId === promptDetails?.prompt_id && updateStatus?.status === promptStudioUpdateStatus.isUpdating) || - disableLlmOrDocChange.includes(promptDetails?.prompt_id) + disableLlmOrDocChange.includes( + promptDetails?.prompt_id + ) || + indexDocs.includes(selectedDoc?.document_id) } > @@ -713,9 +737,11 @@ function PromptCard({ @@ -779,9 +805,10 @@ function PromptCard({ optionFilterProp="children" options={enforceTypeList} value={promptDetails?.enforce_type || null} - disabled={disableLlmOrDocChange.includes( - promptDetails?.prompt_id - )} + disabled={ + disableLlmOrDocChange.includes(promptDetails?.prompt_id) || + indexDocs.includes(selectedDoc?.document_id) + } onChange={(value) => handleTypeChange(value)} />
@@ -818,7 +845,8 @@ function PromptCard({ size="small" disabled={ page <= 1 || - disableLlmOrDocChange.includes(promptDetails?.prompt_id) + disableLlmOrDocChange.includes(promptDetails?.prompt_id) || + indexDocs.includes(selectedDoc?.document_id) } onClick={handlePageLeft} > @@ -829,7 +857,8 @@ function PromptCard({ size="small" disabled={ page >= llmProfiles?.length || - disableLlmOrDocChange.includes(promptDetails?.prompt_id) + disableLlmOrDocChange.includes(promptDetails?.prompt_id) || + indexDocs.includes(selectedDoc?.document_id) } onClick={handlePageRight} > diff --git a/frontend/src/components/custom-tools/text-viewer-pre/TextViewerPre.jsx b/frontend/src/components/custom-tools/text-viewer-pre/TextViewerPre.jsx new file mode 100644 index 000000000..739b2aae4 --- /dev/null +++ b/frontend/src/components/custom-tools/text-viewer-pre/TextViewerPre.jsx @@ -0,0 +1,15 @@ +import PropTypes from "prop-types"; + +function TextViewerPre({ text }) { + return ( +
+ 
{text}
+
+ ); +} + +TextViewerPre.propTypes = { + text: PropTypes.string, +}; + +export { TextViewerPre }; diff --git a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx index 165eb008d..429ceb4cc 100644 --- a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx +++ b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx @@ -12,7 +12,6 @@ import { AddLlmProfileModal } from "../add-llm-profile-modal/AddLlmProfileModal" import { CustomSynonymsModal } from "../custom-synonyms-modal/CustomSynonymsModal"; import { DisplayLogs } from "../display-logs/DisplayLogs"; import { DocumentManager } from "../document-manager/DocumentManager"; -import { GenerateIndex } from "../generate-index/GenerateIndex"; import { Header } from "../header/Header"; import { ManageLlmProfilesModal } from "../manage-llm-profiles-modal/ManageLlmProfilesModal"; import { ToolsMain } from "../tools-main/ToolsMain"; @@ -25,12 +24,15 @@ function ToolIde() { const [openManageLlmModal, setOpenManageLlmModal] = useState(false); const [openAddLlmModal, setOpenAddLlmModal] = useState(false); const [editLlmProfileId, setEditLlmProfileId] = useState(null); - const [isGenerateIndexOpen, setIsGenerateIndexOpen] = useState(false); - const [isGeneratingIndex, setIsGeneratingIndex] = useState(false); - const [generateIndexResult, setGenerateIndexResult] = useState(""); const [modalTitle, setModalTitle] = useState(""); - const { details, updateCustomTool, disableLlmOrDocChange, selectedDoc } = - useCustomToolStore(); + const { + details, + updateCustomTool, + disableLlmOrDocChange, + selectedDoc, + listOfDocs, + indexDocs, + } = useCustomToolStore(); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const axiosPrivate = useAxiosPrivate(); @@ -71,20 +73,21 @@ function ToolIde() { setActiveKey(keys); }; - const handleGenerateIndexModal = (isOpen) => { - if (isGeneratingIndex) { + const generateIndex = async (doc) => { + const docId = doc?.document_id; + const listOfIndexDocs = [...indexDocs]; + + if (listOfIndexDocs.includes(docId)) { + setAlertDetails({ + type: "error", + content: "This document is already getting indexed", + }); return; } - setIsGenerateIndexOpen(isOpen); - }; - - const generateIndex = async (fileName) => { - setIsGenerateIndexOpen(true); - setIsGeneratingIndex(true); const body = { tool_id: details?.tool_id, - file_name: fileName, + document_id: docId, }; const requestOptions = { method: "POST", @@ -96,16 +99,25 @@ function ToolIde() { data: body, }; + listOfIndexDocs.push(docId); + updateCustomTool({ indexDocs: listOfIndexDocs }); return axiosPrivate(requestOptions) .then(() => { - setGenerateIndexResult("SUCCESS"); + setAlertDetails({ + type: "success", + content: `${doc?.document_name} - Indexed successfully`, + }); }) .catch((err) => { - setGenerateIndexResult("FAILED"); - setAlertDetails(handleException(err, "Failed to index")); + setAlertDetails( + handleException(err, `${doc?.document_name} - Failed to index`) + ); }) .finally(() => { - setIsGeneratingIndex(false); + const newListOfIndexDocs = [...indexDocs].filter( + (item) => item !== docId + ); + updateCustomTool({ indexDocs: newListOfIndexDocs }); }); }; @@ -129,7 +141,7 @@ function ToolIde() { }); }; - const handleDocChange = (docName) => { + const handleDocChange = (docId) => { if (disableLlmOrDocChange?.length > 0) { setAlertDetails({ type: "error", @@ -138,14 +150,16 @@ function ToolIde() { return; } + const doc = [...listOfDocs].find((item) => item?.document_id === docId); + const prevSelectedDoc = selectedDoc; const data = { - selectedDoc: docName, + selectedDoc: doc, }; updateCustomTool(data); const body = { - output: docName, + output: docId, }; handleUpdateTool(body).catch((err) => { @@ -228,19 +242,6 @@ function ToolIde() { modalTitle={modalTitle} setModalTitle={setModalTitle} /> - handleGenerateIndexModal(false)} - > - -
); } diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx index 1acff0c47..bb4b38637 100644 --- a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx +++ b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx @@ -136,7 +136,9 @@ function ToolsMain({ setOpenAddLlmModal }) { setScrollToBottom={setScrollToBottom} /> )} - {activeKey === "2" && } + {activeKey === "2" && ( + + )}