diff --git a/.gitignore b/.gitignore index 9b9ca8f4a..9e300b9e3 100644 --- a/.gitignore +++ b/.gitignore @@ -607,6 +607,9 @@ $RECYCLE.BIN/ backend/plugins/authentication/* !backend/plugins/authentication/auth_sample +# Processor Plugins +backend/plugins/processor/* + # Tool registry unstract/tool-registry/src/unstract/tool_registry/*.json unstract/tool-registry/tests/*.yaml @@ -627,5 +630,3 @@ tools/*/sdks/ tools/*/data_dir/ docker/workflow_data/ - - diff --git a/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py new file mode 100644 index 000000000..c0631e723 --- /dev/null +++ b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.1 on 2024-02-28 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("adapter_processor", "0004_alter_adapterinstance_adapter_type"), + ] + + operations = [ + migrations.AlterField( + model_name="adapterinstance", + name="adapter_type", + field=models.CharField( + choices=[ + ("UNKNOWN", "UNKNOWN"), + ("LLM", "LLM"), + ("EMBEDDING", "EMBEDDING"), + ("VECTOR_DB", "VECTOR_DB"), + ("OCR", "OCR"), + ("X2TEXT", "X2TEXT"), + ], + db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB", + ), + ), + ] diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py index a5808f9a5..5c834d852 100644 --- a/backend/file_management/file_management_helper.py +++ b/backend/file_management/file_management_helper.py @@ -195,8 +195,8 @@ def fetch_file_contents( elif file_content_type == "text/plain": with fs.open(file_path, "r") as file: FileManagerHelper.logger.info(f"Reading text file: {file_path}") - encoded_string = base64.b64encode(file.read()) - return encoded_string + text_content = file.read() + return text_content else: raise InvalidFileType @@ -246,9 +246,13 @@ def handle_sub_directory_for_tenants( raise OrgIdNotValid() base_path = Path(settings.PROMPT_STUDIO_FILE_PATH) file_path: Path = base_path / org_id / user_id / tool_id + extract_file_path: Path = Path(file_path, "extract") + summarize_file_path: Path = Path(file_path, "summarize") if is_create: try: os.makedirs(file_path, exist_ok=True) + os.makedirs(extract_file_path, exist_ok=True) + os.makedirs(summarize_file_path, exist_ok=True) except OSError as e: FileManagerHelper.logger.error( f"Error while creating {file_path}: {e}" diff --git a/backend/file_management/views.py b/backend/file_management/views.py index a8756595f..06bb74977 100644 --- a/backend/file_management/views.py +++ b/backend/file_management/views.py @@ -121,7 +121,6 @@ def upload(self, request: HttpRequest) -> Response: @action(detail=True, methods=["post"]) def upload_for_ide(self, request: HttpRequest) -> Response: - print(request.data) serializer = FileUploadIdeSerializer(data=request.data) serializer.is_valid(raise_exception=True) uploaded_files: Any = serializer.validated_data.get("file") diff --git a/backend/pdm.lock b/backend/pdm.lock index da10551b7..9823da806 100644 --- a/backend/pdm.lock +++ b/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "test", "deploy"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:73a791002f5f7bab1afe9eaee07e8fc946774da3448f9daabedbccfc9be4f202" +content_hash = "sha256:27911cc8da87345bc67ffc964493ec9434a770ed2e380403ef87094e8a55e409" [[package]] name = "aiobotocore" @@ -4667,7 +4667,7 @@ dependencies = [ [[package]] name = "unstract-sdk" -version = "0.11.2" +version = "0.12.1" requires_python = "<3.11.1,>=3.9" summary = "A framework for writing Unstract Tools/Apps" groups = ["default"] @@ -4684,8 +4684,8 @@ dependencies = [ "unstract-adapters~=0.2.2", ] files = [ - {file = "unstract_sdk-0.11.2-py3-none-any.whl", hash = "sha256:bcab064d87dc2985e5a4340e438a8ef71ef165b3e5078aba0ed2da3bda77f9e0"}, - {file = "unstract_sdk-0.11.2.tar.gz", hash = "sha256:0d6e3e00c9bbc6e569cc79922a2a45a54870c482cfa61287ea89a22ebd7fc606"}, + {file = "unstract_sdk-0.12.1-py3-none-any.whl", hash = "sha256:960d8a7b92a1ad32894650edd9254aaf7c3f7555053991b3860e55ab684624a8"}, + {file = "unstract_sdk-0.12.1.tar.gz", hash = "sha256:06b1a7bf17948d548137502127458d83046c2e0b596ff326a292dbe94041a13a"}, ] [[package]] @@ -4700,7 +4700,7 @@ dependencies = [ "docker~=6.1.3", "jsonschema~=4.18.2", "unstract-adapters~=0.2.1", - "unstract-tool-sandbox @ file:///home/gayathrivijayakumar/code/Unstract/unstract/unstract/tool-registry/../tool-sandbox", + "unstract-tool-sandbox @ file:///home/ghost/Documents/zipstack/unstract-oss/unstract/tool-registry/../tool-sandbox", ] [[package]] diff --git a/backend/plugins/processor/.gitkeep b/backend/plugins/processor/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.py new file mode 100644 index 000000000..28d12f8d9 --- /dev/null +++ b/backend/prompt_studio/processor_loader.py @@ -0,0 +1,76 @@ +import logging +import os +from importlib import import_module +from typing import Any + +from django.apps import apps + +logger = logging.getLogger(__name__) + + +class ProcessorConfig: + """Loader config for processor plugins.""" + + PLUGINS_APP = "plugins" + PLUGIN_DIR = "processor" + MODULE = "module" + METADATA = "metadata" + METADATA_NAME = "name" + METADATA_SERVICE_CLASS = "service_class" + METADATA_IS_ACTIVE = "is_active" + + +def load_plugins() -> list[Any]: + """Iterate through the processor plugins and register them.""" + plugins_app = apps.get_app_config(ProcessorConfig.PLUGINS_APP) + package_path = plugins_app.module.__package__ + processor_dir = os.path.join(plugins_app.path, ProcessorConfig.PLUGIN_DIR) + processor_package_path = f"{package_path}.{ProcessorConfig.PLUGIN_DIR}" + processor_plugins: list[Any] = [] + + for item in os.listdir(processor_dir): + # Loads a plugin if it is in a directory. + if os.path.isdir(os.path.join(processor_dir, item)): + processor_module_name = item + # Loads a plugin if it is a shared library. + # Module name is extracted from shared library name. + # `processor.platform_architecture.so` will be file name and + # `processor` will be the module name. + elif item.endswith(".so"): + processor_module_name = item.split(".")[0] + else: + continue + try: + full_module_path = ( + f"{processor_package_path}.{processor_module_name}" + ) + module = import_module(full_module_path) + metadata = getattr(module, ProcessorConfig.METADATA, {}) + + if metadata.get(ProcessorConfig.METADATA_IS_ACTIVE, False): + processor_plugins.append( + { + ProcessorConfig.MODULE: module, + ProcessorConfig.METADATA: module.metadata, + } + ) + logger.info( + "Loaded processor plugin: %s, is_active: %s", + module.metadata[ProcessorConfig.METADATA_NAME], + module.metadata[ProcessorConfig.METADATA_IS_ACTIVE], + ) + else: + logger.info( + "Processor plugin %s is not active.", + processor_module_name, + ) + except ModuleNotFoundError as exception: + logger.error( + "Error while importing processor plugin: %s", + exception, + ) + + if len(processor_plugins) == 0: + logger.info("No processor plugins found.") + + return processor_plugins diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py index df5be97e9..a848828af 100644 --- a/backend/prompt_studio/prompt_studio_core/constants.py +++ b/backend/prompt_studio/prompt_studio_core/constants.py @@ -73,6 +73,8 @@ class ToolStudioPromptKeys: EVAL_SETTINGS_EVALUATE = "evaluate" EVAL_SETTINGS_MONITOR_LLM = "monitor_llm" EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed" + SUMMARIZE = "summarize" + SUMMARIZED_RESULT = "summarized_result" class LogLevels: diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0004_customtool_summarize_as_source_and_more.py b/backend/prompt_studio/prompt_studio_core/migrations/0004_customtool_summarize_as_source_and_more.py new file mode 100644 index 000000000..125ca3b2b --- /dev/null +++ b/backend/prompt_studio/prompt_studio_core/migrations/0004_customtool_summarize_as_source_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 4.2.1 on 2024-02-27 05:43 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("prompt_profile_manager", "0006_alter_profilemanager_x2text"), + ("prompt_studio_core", "0003_merge_20240125_1501"), + ] + + operations = [ + migrations.AddField( + model_name="customtool", + name="summarize_as_source", + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name="customtool", + name="summarize_context", + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name="customtool", + name="summarize_llm_profile", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="summarize_llm_profile", + to="prompt_profile_manager.profilemanager", + ), + ), + migrations.AddField( + model_name="customtool", + name="summarize_prompt", + field=models.TextField( + blank=True, db_comment="Field to store the summarize prompt" + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0005_alter_customtool_default_profile_and_more.py b/backend/prompt_studio/prompt_studio_core/migrations/0005_alter_customtool_default_profile_and_more.py new file mode 100644 index 000000000..f8c53210a --- /dev/null +++ b/backend/prompt_studio/prompt_studio_core/migrations/0005_alter_customtool_default_profile_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.1 on 2024-02-28 09:03 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("prompt_profile_manager", "0006_alter_profilemanager_x2text"), + ("prompt_studio_core", "0004_customtool_summarize_as_source_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="customtool", + name="default_profile", + field=models.ForeignKey( + blank=True, + db_comment="Default LLM Profile used in prompt", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="default_profile", + to="prompt_profile_manager.profilemanager", + ), + ), + migrations.AlterField( + model_name="customtool", + name="summarize_as_source", + field=models.BooleanField( + db_comment="Flag to use summarized content as source", + default=True, + ), + ), + migrations.AlterField( + model_name="customtool", + name="summarize_context", + field=models.BooleanField( + db_comment="Flag to summarize content", default=True + ), + ), + migrations.AlterField( + model_name="customtool", + name="summarize_llm_profile", + field=models.ForeignKey( + blank=True, + db_comment="LLM Profile used for summarize", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="summarize_llm_profile", + to="prompt_profile_manager.profilemanager", + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py index 4b8128461..82d41948f 100644 --- a/backend/prompt_studio/prompt_studio_core/models.py +++ b/backend/prompt_studio/prompt_studio_core/models.py @@ -44,6 +44,26 @@ class CustomTool(BaseModel): related_name="default_profile", null=True, blank=True, + db_comment="Default LLM Profile used in prompt", + ) + summarize_llm_profile = models.ForeignKey( + ProfileManager, + on_delete=models.SET_NULL, + related_name="summarize_llm_profile", + null=True, + blank=True, + db_comment="LLM Profile used for summarize", + ) + summarize_context = models.BooleanField( + default=True, db_comment="Flag to summarize content" + ) + summarize_as_source = models.BooleanField( + default=True, db_comment="Flag to use summarized content as source" + ) + summarize_prompt = models.TextField( + blank=True, + db_comment="Field to store the summarize prompt", + unique=False, ) prompt_grammer = models.JSONField( null=True, blank=True, db_comment="Synonymous words used in prompt" diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py index a2a4165a1..33bc189ce 100644 --- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py +++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py @@ -2,7 +2,7 @@ import logging import os from pathlib import Path -from typing import Any, Optional +from typing import Any from django.conf import settings from file_management.file_management_helper import FileManagerHelper @@ -16,7 +16,6 @@ AnswerFetchError, DefaultProfileError, IndexingError, - OutputSaveError, PromptNotValid, ToolNotValid, ) @@ -82,29 +81,39 @@ def fetch_prompt_from_tool(tool_id: str) -> list[ToolStudioPrompt]: @staticmethod def index_document( - tool_id: str, file_name: str, org_id: str, user_id: str + tool_id: str, + file_name: str, + org_id: str, + user_id: str, + is_summary: bool = False, ) -> Any: """Method to index a document. Args: - tool_id (str):Id of the tool + tool_id (str): Id of the tool file_name (str): File to parse + org_id (str): The ID of the organization to which the user belongs. + user_id (str): The ID of the user who uploaded the document. + is_summary (bool, optional): Whether the document is a summary + or not. Defaults to False. Raises: ToolNotValid IndexingError """ tool: CustomTool = CustomTool.objects.get(pk=tool_id) - default_profile: Optional[ProfileManager] = tool.default_profile + if is_summary: + default_profile = tool.summarize_llm_profile + file_path = file_name + else: + default_profile = tool.default_profile + file_path = FileManagerHelper.handle_sub_directory_for_tenants( + org_id, is_create=False, user_id=user_id, tool_id=tool_id + ) + file_path = str(Path(file_path) / file_name) + if not default_profile: raise DefaultProfileError() - file_path = FileManagerHelper.handle_sub_directory_for_tenants( - org_id=org_id, - user_id=user_id, - tool_id=tool_id, - is_create=False, - ) - file_path = str(Path(file_path) / file_name) stream_log.publish( tool_id, stream_log.log( @@ -134,6 +143,7 @@ def index_document( tool_id=tool_id, file_name=file_path, org_id=org_id, + is_summary=is_summary, ) logger.info(f"Indexing done sucessfully for {file_name}") stream_log.publish( @@ -182,6 +192,15 @@ def prompt_responder( prompts: list[ToolStudioPrompt] = [] prompts.append(prompt_instance) tool: CustomTool = prompt_instance.tool_id + + if tool.summarize_as_source: + directory, filename = os.path.split(file_path) + file_path = os.path.join( + directory, + TSPKeys.SUMMARIZE, + os.path.splitext(filename)[0] + ".txt", + ) + stream_log.publish( tool.tool_id, stream_log.log( @@ -277,6 +296,7 @@ def _fetch_response( file_name=path, tool_id=str(tool.tool_id), org_id=org_id, + is_summary=tool.summarize_as_source, ) output: dict[str, Any] = {} @@ -348,19 +368,6 @@ def _fetch_response( if answer["status"] == "ERROR": raise AnswerFetchError() output_response = json.loads(answer["structure_output"]) - - # persist output - try: - for key in output_response: - if TSPKeys.EVAL_RESULT_DELIM not in key: - persisted_prompt = ToolStudioPrompt.objects.get( - prompt_key=key - ) - persisted_prompt.output = output_response[key] or "" - persisted_prompt.save() - except Exception as e: - logger.error(f"Exception while saving prompt output: {e}") - raise OutputSaveError() return output_response @staticmethod @@ -369,6 +376,7 @@ def dynamic_indexer( tool_id: str, file_name: str, org_id: str, + is_summary: bool = False, ) -> str: try: util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id) @@ -380,6 +388,12 @@ def dynamic_indexer( vector_db = str(profile_manager.vector_store.id) x2text_adapter = str(profile_manager.x2text.id) file_hash = ToolUtils.get_hash_from_file(file_path=file_name) + extract_file_path = None + if not is_summary: + directory, filename = os.path.split(file_name) + extract_file_path: str = os.path.join( + directory, "extract", os.path.splitext(filename)[0] + ".txt" + ) return str( tool_index.index_file( tool_id=tool_id, @@ -391,5 +405,6 @@ def dynamic_indexer( chunk_size=profile_manager.chunk_size, chunk_overlap=profile_manager.chunk_overlap, reindex=profile_manager.reindex, + output_file_path=extract_file_path, ) ) diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py index c26844309..5109782d3 100644 --- a/backend/prompt_studio/prompt_studio_core/views.py +++ b/backend/prompt_studio/prompt_studio_core/views.py @@ -6,6 +6,7 @@ from django.db.models import QuerySet from django.http import HttpRequest from permissions.permission import IsOwner +from prompt_studio.processor_loader import ProcessorConfig, load_plugins from prompt_studio.prompt_studio.exceptions import FilenameMissingError from prompt_studio.prompt_studio_core.constants import ( ToolStudioErrors, @@ -42,6 +43,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet): permission_classes = [IsOwner] serializer_class = CustomToolSerializer + processor_plugins = load_plugins() + def get_queryset(self) -> Optional[QuerySet]: filter_args = FilterHelper.build_filter_args( self.request, @@ -141,6 +144,18 @@ def index_document(self, request: HttpRequest) -> Response: org_id=request.org_id, user_id=request.user.user_id, ) + + for processor_plugin in self.processor_plugins: + cls = processor_plugin[ProcessorConfig.METADATA][ + ProcessorConfig.METADATA_SERVICE_CLASS + ] + cls.process( + tool_id=tool_id, + file_name=file_name, + org_id=request.org_id, + user_id=request.user.user_id, + ) + if unique_id: return Response( {"message": "Document indexed successfully."}, diff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0007_promptstudiooutputmanager_eval_metrics.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0007_promptstudiooutputmanager_eval_metrics.py new file mode 100644 index 000000000..b57a9bc6a --- /dev/null +++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0007_promptstudiooutputmanager_eval_metrics.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.1 on 2024-02-27 05:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "prompt_studio_output_manager", + "0006_alter_promptstudiooutputmanager_output", + ), + ] + + operations = [ + migrations.AddField( + model_name="promptstudiooutputmanager", + name="eval_metrics", + field=models.JSONField( + db_column="eval_metrics", + db_comment="Field to store the evaluation metrics", + default=list, + ), + ), + ] diff --git a/backend/prompt_studio/prompt_studio_output_manager/models.py b/backend/prompt_studio/prompt_studio_output_manager/models.py index 3392bd5a2..1e12d7cd2 100644 --- a/backend/prompt_studio/prompt_studio_output_manager/models.py +++ b/backend/prompt_studio/prompt_studio_output_manager/models.py @@ -26,7 +26,13 @@ class PromptStudioOutputManager(BaseModel): db_comment="Field to store the document name", editable=True, ) - + eval_metrics = models.JSONField( + db_column="eval_metrics", + null=False, + blank=False, + default=list, + db_comment="Field to store the evaluation metrics", + ) tool_id = models.ForeignKey( CustomTool, on_delete=models.SET_NULL, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9494a1fda..faf134db2 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "python-socketio==5.9.0", # For log_events "social-auth-app-django==5.3.0", # For OAuth "social-auth-core==4.4.2", # For OAuth - "unstract-sdk~=0.11.2", + "unstract-sdk~=0.12.1", "unstract-adapters~=0.2.2", # ! IMPORTANT! # Indirect local dependencies usually need to be added in their own projects diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx index da3db4600..5340b4bfa 100644 --- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx +++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx @@ -1,4 +1,4 @@ -import { Button, Segmented, Space } from "antd"; +import { Segmented } from "antd"; import jsYaml from "js-yaml"; import Prism from "prismjs"; import "prismjs/components/prism-json"; @@ -6,6 +6,7 @@ import "prismjs/plugins/line-numbers/prism-line-numbers.css"; import "prismjs/plugins/line-numbers/prism-line-numbers.js"; import "prismjs/themes/prism.css"; import { useEffect, useState } from "react"; +import PropTypes from "prop-types"; import { handleException, promptType } from "../../../helpers/GetStaticData"; import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate"; @@ -20,27 +21,31 @@ const outputTypes = { yaml: "YAML", }; -function CombinedOutput() { +function CombinedOutput({ doc, setFilledFields }) { const [combinedOutput, setCombinedOutput] = useState({}); const [yamlData, setYamlData] = useState(null); const [selectedOutputType, setSelectedOutputType] = useState( outputTypes.json ); const [isOutputLoading, setIsOutputLoading] = useState(false); - const { details, selectedDoc } = useCustomToolStore(); + const { details } = useCustomToolStore(); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const axiosPrivate = useAxiosPrivate(); useEffect(() => { - if (!selectedDoc) { + if (!doc) { return; } + let filledFields = 0; setIsOutputLoading(true); handleOutputApiRequest() .then((res) => { const data = res?.data || []; + data.sort((a, b) => { + return new Date(b.modified_at) - new Date(a.modified_at); + }); const prompts = details?.prompts; const output = {}; prompts.forEach((item) => { @@ -59,10 +64,18 @@ function CombinedOutput() { return; } - output[item?.prompt_key] = outputDetails?.output || ""; + output[item?.prompt_key] = JSON.parse(outputDetails?.output || ""); + + if (outputDetails?.output?.length > 0) { + filledFields++; + } }); setCombinedOutput(output); + if (setFilledFields) { + setFilledFields(filledFields); + } + const yamlDump = jsYaml.dump(output); setYamlData(yamlDump); }) @@ -74,7 +87,7 @@ function CombinedOutput() { .finally(() => { setIsOutputLoading(false); }); - }, [selectedDoc]); + }, [doc]); useEffect(() => { Prism.highlightAll(); @@ -87,7 +100,7 @@ function CombinedOutput() { const handleOutputApiRequest = async () => { const requestOptions = { method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&doc_name=${selectedDoc}`, + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&doc_name=${doc}`, headers: { "X-CSRFToken": sessionDetails?.csrfToken, }, @@ -107,11 +120,6 @@ function CombinedOutput() { return (
- -
- -
-
{ + const fileNameTxt = removeFileExtension(selectedDoc); + const files = [ + { + fileName: selectedDoc, + viewType: viewTypes.pdf, + }, + { + fileName: `extract/${fileNameTxt}.txt`, + viewType: viewTypes.extract, + }, + { + fileName: `summarize/${fileNameTxt}.txt`, + viewType: viewTypes.summarize, + }, + ]; + + setFileUrl(""); + setExtractTxt(""); + setSummaryTxt(""); + files.forEach((item) => { + handleFetchContent(item); + }); + }, [selectedDoc]); + + const handleFetchContent = (fileDetails) => { + if (!selectedDoc) { + setFileUrl(""); + setExtractTxt(""); + setSummaryTxt(""); + return; + } + + const requestOptions = { + method: "GET", + url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?file_name=${fileDetails?.fileName}&tool_id=${details?.tool_id}`, + }; + + handleLoadingStateUpdate(fileDetails?.viewType, true); + axiosPrivate(requestOptions) + .then((res) => { + const data = res?.data?.data; + if (fileDetails?.viewType === viewTypes.pdf) { + const base64String = data || ""; + const blob = base64toBlob(base64String); + setFileUrl(URL.createObjectURL(blob)); + return; + } + + if (fileDetails?.viewType === viewTypes?.extract) { + setExtractTxt(data); + } + + if (fileDetails?.viewType === viewTypes.summarize) { + setSummaryTxt(data); + } + }) + .catch((err) => {}) + .finally(() => { + handleLoadingStateUpdate(fileDetails?.viewType, false); + }); + }; + + const handleLoadingStateUpdate = (viewType, value) => { + if (viewType === viewTypes.pdf) { + setIsDocLoading(value); + } + + if (viewType === viewTypes.extract) { + setIsExtractLoading(value); + } + + if (viewType === viewTypes.summarize) { + setIsSummaryLoading(value); + } + }; + + const handleActiveKeyChange = (key) => { + setActiveKey(key); + }; useEffect(() => { const index = [...listOfDocs].findIndex((item) => item === selectedDoc); @@ -49,42 +168,82 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) { return (
-
- -
-
- - +
+
+ +
+ + + +
+
+ + +
+
- + {activeKey === "1" && ( + 0} + setOpenManageDocsModal={setOpenManageDocsModal} + > + + + )} + {activeKey === "2" && ( + 0} + setOpenManageDocsModal={setOpenManageDocsModal} + > + + + )} + {activeKey === "3" && ( + 0} + setOpenManageDocsModal={setOpenManageDocsModal} + > + + + )} { const promptsAndNotes = details?.prompts || []; let name = ""; @@ -93,21 +94,23 @@ function DocumentParser({ promptId, promptStudioUpdateStatus.isUpdating ); - axiosPrivate(requestOptions) + + return axiosPrivate(requestOptions) .then((res) => { const data = res?.data; const modifiedDetails = { ...details }; const modifiedPrompts = [...(modifiedDetails?.prompts || [])].map( (item) => { if (item?.prompt_id === data?.prompt_id) { - data.evalMetrics = item?.evalMetrics || []; return data; } return item; } ); modifiedDetails["prompts"] = modifiedPrompts; - updateCustomTool({ details: modifiedDetails }); + if (!isPromptUpdate) { + updateCustomTool({ details: modifiedDetails }); + } handleUpdateStatus( isUpdateStatus, promptId, diff --git a/frontend/src/components/custom-tools/document-viewer/DocumentViewer.jsx b/frontend/src/components/custom-tools/document-viewer/DocumentViewer.jsx new file mode 100644 index 000000000..aabd40c63 --- /dev/null +++ b/frontend/src/components/custom-tools/document-viewer/DocumentViewer.jsx @@ -0,0 +1,46 @@ +import PropTypes from "prop-types"; +import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader"; +import { EmptyState } from "../../widgets/empty-state/EmptyState"; +import { Typography } from "antd"; + +function DocumentViewer({ + children, + doc, + isLoading, + isContentAvailable, + setOpenManageDocsModal, +}) { + if (isLoading) { + return ; + } + + if (!doc) { + return ( + setOpenManageDocsModal(true)} + /> + ); + } + + if (!isContentAvailable) { + return ( +
+ Failed to load the document +
+ ); + } + + return <>{children}; +} + +DocumentViewer.propTypes = { + children: PropTypes.any.isRequired, + doc: PropTypes.string, + isLoading: PropTypes.bool.isRequired, + isContentAvailable: PropTypes.bool.isRequired, + setOpenManageDocsModal: PropTypes.func, +}; + +export { DocumentViewer }; diff --git a/frontend/src/components/custom-tools/editable-text/EditableText.jsx b/frontend/src/components/custom-tools/editable-text/EditableText.jsx index ac5b14b00..6ba422c94 100644 --- a/frontend/src/components/custom-tools/editable-text/EditableText.jsx +++ b/frontend/src/components/custom-tools/editable-text/EditableText.jsx @@ -15,6 +15,8 @@ function EditableText({ isTextarea, }) { const [text, setText] = useState(""); + const name = isTextarea ? "prompt" : "prompt_key"; + const [triggerHandleChange, setTriggerHandleChange] = useState(false); const [isHovered, setIsHovered] = useState(false); const divRef = useRef(null); const { disableLlmOrDocChange } = useCustomToolStore(); @@ -45,11 +47,19 @@ function EditableText({ const onSearchDebounce = useCallback( debounce((event) => { - handleChange(event, promptId, false, true); + setTriggerHandleChange(true); }, 1000), [] ); + useEffect(() => { + if (!triggerHandleChange) { + return; + } + handleChange(text, promptId, name, true, true); + setTriggerHandleChange(false); + }, [triggerHandleChange]); + const handleClickOutside = (event) => { if (divRef.current && !divRef.current.contains(event.target)) { // Clicked outside the div @@ -64,7 +74,7 @@ function EditableText({ value={text} onChange={handleTextChange} placeholder="Enter Prompt" - name="prompt" + name={name} size="small" style={{ backgroundColor: "transparent" }} variant={`${!isEditing && !isHovered ? "borderless" : "outlined"}`} @@ -84,7 +94,7 @@ function EditableText({ value={text} onChange={handleTextChange} placeholder="Enter Key" - name="prompt_key" + name={name} size="small" style={{ backgroundColor: "transparent" }} variant={`${!isEditing && !isHovered ? "borderless" : "outlined"}`} diff --git a/frontend/src/components/custom-tools/header/Header.jsx b/frontend/src/components/custom-tools/header/Header.jsx index 498512cd8..355e7de10 100644 --- a/frontend/src/components/custom-tools/header/Header.jsx +++ b/frontend/src/components/custom-tools/header/Header.jsx @@ -4,13 +4,12 @@ import { DiffOutlined, EditOutlined, ExportOutlined, - FilePdfOutlined, FileTextOutlined, MessageOutlined, } from "@ant-design/icons"; import { Button, Tooltip, Typography } from "antd"; import PropTypes from "prop-types"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; import "./Header.css"; @@ -21,26 +20,34 @@ import { useCustomToolStore } from "../../../store/custom-tool-store"; import { useSessionStore } from "../../../store/session-store"; import { CustomButton } from "../../widgets/custom-button/CustomButton"; import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal"; +import { SelectLlmProfileModal } from "../select-llm-profile-modal/SelectLlmProfileModal"; function Header({ setOpenCusSynonymsModal, - setOpenManageDocsModal, setOpenManageLlmModal, handleUpdateTool, }) { const [openPreOrPostAmbleModal, setOpenPreOrPostAmbleModal] = useState(false); + const [openSummLlmProfileModal, setOpenSummLlmProfileModal] = useState(false); const [preOrPostAmble, setPreOrPostAmble] = useState(""); const [isExportLoading, setIsExportLoading] = useState(false); - const { details } = useCustomToolStore(); + const [summarizeLlmBtnText, setSummarizeLlmBtnText] = useState(null); + const [llmItems, setLlmItems] = useState([]); + const { details, llmProfiles } = useCustomToolStore(); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const axiosPrivate = useAxiosPrivate(); const navigate = useNavigate(); + useEffect(() => { + getLlmProfilesDropdown(); + }, []); + const handleOpenPreOrPostAmbleModal = (type) => { setOpenPreOrPostAmbleModal(true); setPreOrPostAmble(type); }; + const handleClosePreOrPostAmbleModal = () => { setOpenPreOrPostAmbleModal(false); setPreOrPostAmble(""); @@ -68,6 +75,16 @@ function Header({ }); }; + const getLlmProfilesDropdown = () => { + const items = [...llmProfiles].map((item) => { + return { + value: item?.profile_id, + label: item?.profile_name, + }; + }); + setLlmItems(items); + }; + return (
@@ -89,25 +106,31 @@ function Header({
- - - +
- - - +
- - - +
@@ -143,13 +166,19 @@ function Header({ type={preOrPostAmble} handleUpdateTool={handleUpdateTool} /> +
); } Header.propTypes = { setOpenCusSynonymsModal: PropTypes.func.isRequired, - setOpenManageDocsModal: PropTypes.func.isRequired, setOpenManageLlmModal: PropTypes.func.isRequired, handleUpdateTool: PropTypes.func.isRequired, }; diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx index 7630be445..87e97a980 100644 --- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx +++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx @@ -94,7 +94,27 @@ function ManageDocsModal({ setRows(newRows); }, [listOfDocs, selectedDoc, disableLlmOrDocChange]); - const handleUploadChange = (info) => { + const beforeUpload = (file) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => { + const fileName = file.name; + const fileAlreadyExists = [...listOfDocs].includes(fileName); + if (!fileAlreadyExists) { + resolve(file); + } else { + setAlertDetails({ + type: "error", + content: "File name already exists", + }); + reject(new Error("File name already exists")); + } + }; + }); + }; + + const handleUploadChange = async (info) => { if (info.file.status === "uploading") { setIsUploading(true); } @@ -109,14 +129,14 @@ function ManageDocsModal({ const docName = info?.file?.name; const newListOfDocs = [...listOfDocs]; newListOfDocs.push(docName); + setOpen(false); + await generateIndex(info?.file?.name); const body = { selectedDoc: docName, listOfDocs: newListOfDocs, }; updateCustomTool(body); handleUpdateTool({ output: docName }); - setOpen(false); - generateIndex(info?.file?.name); } else if (info.file.status === "error") { setIsUploading(false); setAlertDetails({ @@ -175,6 +195,8 @@ function ManageDocsModal({ onChange={handleUploadChange} disabled={isUploading || !defaultLlmProfile} showUploadList={false} + accept=".pdf" + beforeUpload={beforeUpload} > + {updateStatus?.promptId === details?.prompt_id && ( + <> + {updateStatus?.status === + promptStudioUpdateStatus.isUpdating && ( + } + color="processing" + className="display-flex-align-center" + > + Updating + + )} + {updateStatus?.status === promptStudioUpdateStatus.done && ( + } + color="success" + className="display-flex-align-center" + > + Done + + )} + + )} + + Output Viewer + + +
+
+
+ ); +} + +export { OutputAnalyzerHeader }; diff --git a/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.css b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.css new file mode 100644 index 000000000..358230c07 --- /dev/null +++ b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.css @@ -0,0 +1,71 @@ +/* Styles for OutputAnalyzerList */ + +.output-analyzer-layout { + height: 100%; + display: flex; + flex-direction: column; + background-color: var(--page-bg-2); + overflow-y: hidden; +} + +.output-analyzer-header { + padding: 8px; + background-color: #F5F7F9; + display: flex; + justify-content: space-between; + align-items: center; +} + +.output-analyzer-header .title { + font-size: 16px; +} + +.output-analyzer-body { + padding: 12px; + height: 100%; + overflow-y: auto; +} + +.output-analyzer-body2 { + height: 700px; + display: flex; + flex-direction: column; +} + +.output-analyzer-card-head { + padding: 12px; + display: flex; + justify-content: space-between; + background-color: #ECEFF3; +} + +.output-analyzer-main { + flex: 1; + overflow-y: hidden; +} + +.output-analyzer-left-box { + padding-right: 6px; + height: 100%; + overflow-y: auto; +} + +.output-analyzer-left-box > div { + height: 100%; + background-color: var(--white); +} + +.output-analyzer-right-box { + padding-left: 6px; + height: 100%; +} + +.output-analyzer-right-box > div { + height: 100%; + background-color: var(--white); + padding: 0px 12px; +} + +.output-analyzer-card-gap { + margin-bottom: 12px; +} \ No newline at end of file diff --git a/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx new file mode 100644 index 000000000..7b8b3d392 --- /dev/null +++ b/frontend/src/components/custom-tools/output-analyzer-list/OutputAnalyzerList.jsx @@ -0,0 +1,38 @@ +import { OutputAnalyzerHeader } from "../output-analyzer-header/OutputAnalyzerHeader"; +import "./OutputAnalyzerList.css"; +import { OutputAnalyzerCard } from "../output-analyzer-card/OutputAnalyzerCard"; +import { useCustomToolStore } from "../../../store/custom-tool-store"; +import { useEffect, useState } from "react"; +import { promptType } from "../../../helpers/GetStaticData"; + +function OutputAnalyzerList() { + const [totalFields, setTotalFields] = useState(0); + const { listOfDocs, details } = useCustomToolStore(); + + useEffect(() => { + const prompts = [...(details?.prompts || [])]; + const promptsFiltered = prompts.filter( + (item) => item?.prompt_type === promptType.prompt + ); + setTotalFields(promptsFiltered.length || 0); + }, [details]); + + return ( +
+
+ +
+
+ {listOfDocs.map((doc) => { + return ( +
+ +
+ ); + })} +
+
+ ); +} + +export { OutputAnalyzerList }; diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx index 1bfb89d5d..8bc293ddf 100644 --- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx +++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx @@ -7,6 +7,8 @@ import { useSessionStore } from "../../../store/session-store"; import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate"; import "./OutputForDocModal.css"; import { CheckCircleFilled, CloseCircleFilled } from "@ant-design/icons"; +import { useNavigate } from "react-router-dom"; +import { displayPromptResult } from "../../../helpers/GetStaticData"; const columns = [ { @@ -32,6 +34,8 @@ function OutputForDocModal({ const { details, listOfDocs } = useCustomToolStore(); const { sessionDetails } = useSessionStore(); const axiosPrivate = useAxiosPrivate(); + const navigate = useNavigate(); + useEffect(() => { handleGetOutputForDocs(); }, [open]); @@ -53,7 +57,7 @@ function OutputForDocModal({ .then((res) => { const data = res?.data || []; data.sort((a, b) => { - return new Date(b.created_at) - new Date(a.created_at); + return new Date(b.modified_at) - new Date(a.modified_at); }); handleRowsGeneration(data); }) @@ -67,7 +71,6 @@ function OutputForDocModal({ [...listOfDocs].forEach((item) => { const output = data.find((outputValue) => outputValue?.doc_name === item); const isSuccess = output?.output?.length > 0; - const content = isSuccess ? output?.output : "Failed"; const result = { key: item, @@ -81,7 +84,7 @@ function OutputForDocModal({ )} {" "} - {content} + {isSuccess ? displayPromptResult(output?.output) : "Failed"} ), }; @@ -108,7 +111,9 @@ function OutputForDocModal({
- +
diff --git a/frontend/src/components/custom-tools/pdf-viewer/PdfViewer.jsx b/frontend/src/components/custom-tools/pdf-viewer/PdfViewer.jsx index 5fd9f9201..4ee872189 100644 --- a/frontend/src/components/custom-tools/pdf-viewer/PdfViewer.jsx +++ b/frontend/src/components/custom-tools/pdf-viewer/PdfViewer.jsx @@ -1,87 +1,11 @@ import { Viewer, Worker } from "@react-pdf-viewer/core"; import { defaultLayoutPlugin } from "@react-pdf-viewer/default-layout"; import { pageNavigationPlugin } from "@react-pdf-viewer/page-navigation"; -import { Typography } from "antd"; import PropTypes from "prop-types"; -import { useEffect, useState } from "react"; -import { handleException } from "../../../helpers/GetStaticData"; -import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate"; -import { useAlertStore } from "../../../store/alert-store"; -import { useCustomToolStore } from "../../../store/custom-tool-store"; -import { useSessionStore } from "../../../store/session-store"; -import { EmptyState } from "../../widgets/empty-state/EmptyState"; -import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader"; - -function PdfViewer({ setOpenManageDocsModal }) { - const [fileUrl, setFileUrl] = useState(""); - const [isLoading, setIsLoading] = useState(false); +function PdfViewer({ fileUrl }) { const newPlugin = defaultLayoutPlugin(); const pageNavigationPluginInstance = pageNavigationPlugin(); - const { sessionDetails } = useSessionStore(); - const { setAlertDetails } = useAlertStore(); - const { selectedDoc, details } = useCustomToolStore(); - const axiosPrivate = useAxiosPrivate(); - - const base64toBlob = (data) => { - const bytes = atob(data); - let length = bytes.length; - const out = new Uint8Array(length); - - while (length--) { - out[length] = bytes.charCodeAt(length); - } - - return new Blob([out], { type: "application/pdf" }); - }; - - useEffect(() => { - if (!selectedDoc) { - setFileUrl(""); - return; - } - - const requestOptions = { - method: "GET", - url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/file/fetch_contents?file_name=${selectedDoc}&tool_id=${details?.tool_id}`, - }; - - setIsLoading(true); - axiosPrivate(requestOptions) - .then((res) => { - const base64String = res?.data?.data || ""; - const blob = base64toBlob(base64String); - setFileUrl(URL.createObjectURL(blob)); - }) - .catch((err) => { - setAlertDetails(handleException(err, "Failed to load the document")); - }) - .finally(() => { - setIsLoading(false); - }); - }, [selectedDoc]); - - if (isLoading) { - return ; - } - - if (!selectedDoc) { - return ( - setOpenManageDocsModal(true)} - /> - ); - } - - if (!fileUrl) { - return ( -
- Failed to load the document -
- ); - } return (
@@ -96,7 +20,7 @@ function PdfViewer({ setOpenManageDocsModal }) { } PdfViewer.propTypes = { - setOpenManageDocsModal: PropTypes.func.isRequired, + fileUrl: PropTypes.any, }; export { PdfViewer }; diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx index 9e5eebcb5..4b6af7cbc 100644 --- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx +++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx @@ -4,6 +4,7 @@ import { DeleteOutlined, EditOutlined, LeftOutlined, + LoadingOutlined, PlayCircleOutlined, RightOutlined, SearchOutlined, @@ -30,6 +31,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { AssertionIcon } from "../../../assets"; import { + displayPromptResult, handleException, promptStudioUpdateStatus, } from "../../../helpers/GetStaticData"; @@ -63,6 +65,7 @@ function PromptCard({ const [result, setResult] = useState({ promptOutputId: null, output: "", + evalMetrics: [], }); const [outputIds, setOutputIds] = useState([]); const [coverage, setCoverage] = useState(0); @@ -75,7 +78,6 @@ function PromptCard({ llmProfiles, selectedDoc, listOfDocs, - evalMetrics, updateCustomTool, details, disableLlmOrDocChange, @@ -174,10 +176,6 @@ function PromptCard({ [] ); - const isJson = (text) => { - return typeof text === "object"; - }; - const handlePageLeft = () => { if (page <= 1) { return; @@ -214,24 +212,16 @@ function PromptCard({ return sortedMetrics; }; + const handleTypeChange = (value) => { + handleChange(value, promptDetails?.prompt_id, "enforce_type", true).then( + () => { + handleRun(); + } + ); + }; + // Generate the result for the currently selected document const handleRun = () => { - if (!promptDetails?.prompt_key) { - setAlertDetails({ - type: "error", - content: "Prompt key is not set", - }); - return; - } - - if (!promptDetails?.prompt) { - setAlertDetails({ - type: "error", - content: "Prompt cannot be empty", - }); - return; - } - if (!promptDetails?.profile_manager?.length) { setAlertDetails({ type: "error", @@ -251,41 +241,39 @@ function PromptCard({ setIsRunLoading(true); setIsCoverageLoading(true); setCoverage(0); + setCoverageTotal(0); + + let method = "POST"; + let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`; + if (result?.promptOutputId) { + method = "PATCH"; + url += `${result?.promptOutputId}/`; + } handleRunApiRequest(selectedDoc) .then((res) => { const data = res?.data; const value = data[promptDetails?.prompt_key]; - let method = "POST"; - let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`; - if (result?.promptOutputId) { - method = "PATCH"; - url += `${result?.promptOutputId}/`; - } - handleUpdateOutput(value, selectedDoc, method, url); - - if (value?.length > 0) { + if (value !== null && String(value)?.length > 0) { setCoverage((prev) => prev + 1); - } else { - setAlertDetails({ - type: "error", - content: `Failed to generate output for ${selectedDoc}`, - }); } - handleCoverage(); - const modifiedEvalMetrics = { ...evalMetrics }; - modifiedEvalMetrics[`${promptDetails?.prompt_id}`] = - sortEvalMetricsByType( - data[`${promptDetails?.prompt_key}__evaluation`] || [] - ); - updateCustomTool({ evalMetrics: modifiedEvalMetrics }); + // Handle Eval + let evalMetrics = []; + if (promptDetails?.evaluate) { + evalMetrics = data[`${promptDetails?.prompt_key}__evaluation`] || []; + } + handleUpdateOutput(value, selectedDoc, evalMetrics, method, url); }) .catch((err) => { - setIsCoverageLoading(false); - setAlertDetails(handleException(err)); + handleUpdateOutput(null, selectedDoc, [], method, url); + setAlertDetails( + handleException(err, `Failed to generate output for ${selectedDoc}`) + ); }) .finally(() => { setIsRunLoading(false); + setCoverageTotal((prev) => prev + 1); + handleCoverage(); }); }; @@ -300,14 +288,32 @@ function PromptCard({ return; } - setCoverageTotal(1); listOfDocsToProcess.forEach((item) => { + let method = "POST"; + let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`; + const outputId = outputIds.find((output) => output?.docName === item); + if (outputId?.promptOutputId?.length) { + method = "PATCH"; + url += `${outputId?.promptOutputId}/`; + } handleRunApiRequest(item) .then((res) => { const data = res?.data; - handleCoverageData(data, item); + const outputValue = data[promptDetails?.prompt_key]; + if (outputValue !== null && String(outputValue)?.length > 0) { + setCoverage((prev) => prev + 1); + } + + // Handle Eval + let evalMetrics = []; + if (promptDetails?.evaluate) { + evalMetrics = + data[`${promptDetails?.prompt_key}__evaluation`] || []; + } + handleUpdateOutput(outputValue, item, evalMetrics, method, url); }) .catch((err) => { + handleUpdateOutput(null, item, [], method, url); setAlertDetails( handleException(err, `Failed to generate output for ${item}`) ); @@ -344,34 +350,20 @@ function PromptCard({ }); }; - const handleCoverageData = (data, docName) => { - const outputValue = data[promptDetails?.prompt_key]; - let method = "POST"; - let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`; - const outputId = outputIds.find((output) => output?.docName === docName); - if (outputId?.promptOutputId?.length) { - method = "PATCH"; - url += `${outputId?.promptOutputId}/`; - } - handleUpdateOutput(outputValue, docName, method, url); - - if (outputValue?.length > 0) { - setCoverage((prev) => prev + 1); - } else { - setAlertDetails({ - type: "error", - content: `Failed to generate output for ${docName}`, - }); - } - }; - - const handleUpdateOutput = (outputValue, docName, method, url) => { + const handleUpdateOutput = ( + outputValue, + docName, + evalMetrics, + method, + url + ) => { const body = { - output: outputValue, + output: outputValue !== null ? JSON.stringify(outputValue) : null, tool_id: details?.tool_id, prompt_id: promptDetails?.prompt_id, profile_manager: promptDetails?.profile_manager, doc_name: docName, + eval_metrics: evalMetrics, }; const requestOptions = { @@ -386,14 +378,24 @@ function PromptCard({ axiosPrivate(requestOptions) .then((res) => { - if (docName !== selectedDoc) { - return; - } const data = res?.data; - setResult({ - promptOutputId: data?.prompt_output_id || null, - output: data?.output || "", - }); + const promptOutputId = data?.prompt_output_id || null; + if (docName === selectedDoc) { + setResult({ + promptOutputId: promptOutputId, + output: data?.output, + evalMetrics: sortEvalMetricsByType(data?.eval_metrics || []), + }); + } + + const isOutputIdAvailable = outputIds.find( + (item) => item?.promptOutputId === promptOutputId + ); + if (!isOutputIdAvailable) { + const listOfOutputIds = [...outputIds]; + listOfOutputIds.push({ promptOutputId, docName }); + setOutputIds(listOfOutputIds); + } }) .catch((err) => { setAlertDetails(handleException(err, "Failed to persist the result")); @@ -405,19 +407,19 @@ function PromptCard({ setResult({ promptOutputId: null, output: "", + evalMetrics: [], }); return; } setIsRunLoading(true); handleOutputApiRequest(true) - .then((res) => { - const data = res?.data; - + .then((data) => { if (!data || data?.length === 0) { setResult({ promptOutputId: null, output: "", + evalMetrics: [], }); return; } @@ -425,7 +427,8 @@ function PromptCard({ const outputResult = data[0]; setResult({ promptOutputId: outputResult?.prompt_output_id, - output: outputResult?.output || "", + output: outputResult?.output, + evalMetrics: sortEvalMetricsByType(outputResult?.eval_metrics || []), }); }) .catch((err) => { @@ -443,11 +446,7 @@ function PromptCard({ setCoverage(0); handleOutputApiRequest() - .then((res) => { - const data = res?.data; - data.sort((a, b) => { - return new Date(b.created_at) - new Date(a.created_at); - }); + .then((data) => { handleGetCoverageData(data); }) .catch((err) => { @@ -470,7 +469,13 @@ function PromptCard({ }; return axiosPrivate(requestOptions) - .then((res) => res) + .then((res) => { + const data = res?.data; + data.sort((a, b) => { + return new Date(b.modified_at) - new Date(a.modified_at); + }); + return data; + }) .catch((err) => { throw err; }); @@ -488,7 +493,7 @@ function PromptCard({ } if ( - item?.output?.length > 0 && + item?.output !== undefined && [...listOfDocs].includes(item?.doc_name) ) { ids.push({ @@ -581,6 +586,15 @@ function PromptCard({ /> + {isCoverageLoading && ( + } + color="processing" + className="display-flex-align-center" + > + Generating Response + + )} {updateStatus?.promptId === promptDetails?.prompt_id && ( <> {updateStatus?.status === @@ -680,8 +694,10 @@ function PromptCard({ 0) && - "prompt-card-comp-layout-border" + !( + isRunLoading || + (result?.output !== undefined && outputIds?.length > 0) + ) && "prompt-card-comp-layout-border" }`} >
@@ -708,6 +724,7 @@ function PromptCard({ icon={} loading={isCoverageLoading} onClick={() => setOpenOutputForDoc(true)} + disabled={outputIds?.length === 0} > Coverage: {coverage} of {listOfDocs?.length || 0} docs @@ -725,14 +742,7 @@ function PromptCard({ disabled={disableLlmOrDocChange.includes( promptDetails?.prompt_id )} - onChange={(value) => - handleChange( - value, - promptDetails?.prompt_id, - "enforce_type", - true - ) - } + onChange={(value) => handleTypeChange(value)} />
@@ -781,18 +791,17 @@ function PromptCard({
- {evalMetrics?.[`${promptDetails?.prompt_id}`] && ( + {result?.evalMetrics?.length > 0 && (
- {evalMetrics?.[`${promptDetails?.prompt_id}`].map( - (evalMetric) => ( - - ) - )} + {result?.evalMetrics.map((evalMetric) => ( + + ))}
)} - {(isRunLoading || result?.output?.length > 0) && ( + {(isRunLoading || + (result?.output !== undefined && outputIds?.length > 0)) && ( <>
@@ -800,11 +809,7 @@ function PromptCard({ } /> ) : ( -
- {isJson(result?.output) - ? JSON.stringify(result?.output, null, 4) - : result?.output} -
+
{displayPromptResult(result?.output)}
 )}
@@ -816,7 +821,6 @@ function PromptCard({ setOpen={setOpenEval} promptDetails={promptDetails} handleChange={handleChange} - handleRun={handleRun} /> { + setIsContext(details?.summarize_context); + setIsSource(details?.summarize_as_source); + }, []); + + useEffect(() => { + if (!selectedLlm) { + setBtnText(""); + + // If the LLM is not selected, the context needs to be set to false and disabled in the UI + if (isContext) { + handleLlmProfileChange(fieldNames.SUMMARIZE_CONTEXT, false); + } + return; + } + + const llmItem = [...llmItems].find((item) => item?.value === selectedLlm); + setBtnText(llmItem?.label || ""); + }, [selectedLlm]); + + useEffect(() => { + if (llmItems?.length) { + setSelectedLlm(details?.summarize_llm_profile); + } + }, [llmItems]); + + const handleStateUpdate = (fieldName, value) => { + if (fieldName === fieldNames.SUMMARIZE_LLM_PROFILE) { + setSelectedLlm(value); + } + + if (fieldName === fieldNames.SUMMARIZE_PROMPT) { + setPrompt(value); + } + + if (fieldName === fieldNames.SUMMARIZE_CONTEXT) { + setIsContext(value); + } + + if (fieldName === fieldNames.SUMMARIZE_AS_SOURCE) { + setIsSource(value); + } + }; + + const handleLlmProfileChange = (fieldName, value) => { + handleStateUpdate(fieldName, value); + const body = { + [fieldName]: value, + }; + + if (fieldName === fieldNames.SUMMARIZE_CONTEXT && !value) { + body[fieldNames.SUMMARIZE_AS_SOURCE] = false; + handleStateUpdate(fieldNames.SUMMARIZE_AS_SOURCE, false); + } + + handleUpdateTool(body) + .then(() => { + setAlertDetails({ + type: "success", + content: "Successfully updated the LLM profile", + }); + }) + .catch((err) => { + setAlertDetails( + handleException(err, "Failed to update the LLM profile") + ); + }); + }; + + const onSearchDebounce = useCallback( + debounce((event) => { + handleLlmProfileChange(event.target.name, event.target.value); + }, 1000), + [] + ); + + return ( + setOpen(null)} + maskClosable={false} + centered + footer={null} + > +
+
+ + Summarize Manager + +
+
+ +