Merge pull request #88 from UN-OCHA/develop

Main -> Develop - 1.6.0

config/install/ocha_ai.settings.yml

@@ -1,4 +1,9 @@
 plugins:
+  answer_validator:
+    similarity_embedding:
+      min_similarity: 0.2
+    similarity_ranker:
+      min_similarity: 0.8
   completion:
     aws_bedrock:
       prompt_template: |-
@@ -12,6 +17,10 @@ plugins:
         Based solely on the information below, please answer the user's question. Please do not make things up and say you don't know if you cannot answer.
 
         {{ context }}
+  ranker:
+    ocha_ai_helper_ranker:
+      endpoint: http://ocha-ai-helper/text/correlate/texts
+      limit: 5
   source:
     reliefweb:
       api_url: https://api.reliefweb.int/v1
@@ -38,10 +47,14 @@ plugins:
       nested_object_limit: 100000
 defaults:
   plugins:
+    answer_validator:
+      plugin_id: similarity_embedding
     completion:
       plugin_id: aws_bedrock
     embedding:
       plugin_id: aws_bedrock
+    ranker:
+      plugin_id: ocha_ai_helper_ranker
     source:
       plugin_id: reliefweb
     text_extractor:

config/schema/ocha_ai.schema.yml

@@ -6,6 +6,12 @@ ocha_ai.settings:
       type: mapping
       label: 'List of plugins.'
       mapping:
+        answer_validator:
+          type: sequence
+          label: 'List of answer validator plugins.'
+          sequence:
+            type: ocha_ai.plugin.answer_validator.[%key]
+            label: 'Settings for a answer validator plugin.'
         completion:
           type: sequence
           label: 'List of completion plugins.'
@@ -18,6 +24,12 @@ ocha_ai.settings:
           sequence:
             type: ocha_ai.plugin.embedding.[%key]
             label: 'Settings for a embedding plugin.'
+        ranker:
+          type: sequence
+          label: 'List of ranker plugins.'
+          sequence:
+            type: ocha_ai.plugin.ranker.[%key]
+            label: 'Settings for a ranker plugin.'
         source:
           type: sequence
           label: 'List of source plugins.'
@@ -57,6 +69,13 @@ ocha_ai.settings:
           type: mapping
           label: 'Default plugins.'
           mapping:
+            answer_validator:
+              type: mapping
+              label: 'Default answer validator plugin settings.'
+              mapping:
+                plugin_id:
+                  type: string
+                  label: 'Plugin ID.'
             completion:
               type: mapping
               label: 'Default completion plugin settings.'
@@ -71,6 +90,13 @@ ocha_ai.settings:
                 plugin_id:
                   type: string
                   label: 'Plugin ID.'
+            ranker:
+              type: mapping
+              label: 'Default ranker plugin settings.'
+              mapping:
+                plugin_id:
+                  type: string
+                  label: 'Plugin ID.'
             source:
               type: mapping
               label: 'Default source plugin settings.'
@@ -105,6 +131,12 @@ ocha_ai.settings:
 
 ### Base plugin settings. ###
 
+# Ranker plugin base settings.
+ocha_ai.plugin.answer_validation:
+  type: mapping
+  label: 'Answer validator plugin base settings.'
+  mapping:
+
 # Completion plugin base settings.
 ocha_ai.plugin.completion:
   type: mapping
@@ -162,6 +194,15 @@ ocha_ai.plugin.embedding:
       type: integer
       label: 'Max tokens.'
 
+# Ranker plugin base settings.
+ocha_ai.plugin.ranker:
+  type: mapping
+  label: 'Ranker plugin base settings.'
+  mapping:
+    limit:
+      type: integer
+      label: 'Maximum number of relevant texts to return.'
+
 # Source plugin base settings.
 ocha_ai.plugin.source:
   type: mapping
@@ -194,6 +235,24 @@ ocha_ai.plugin.vector_store:
 
 ### Plugin settings. ###
 
+# Similarity - Embedding answer validator plugin settings.
+ocha_ai.plugin.answer_validator.similarity_embedding:
+  type: mapping
+  label: 'Similarity - Embedding answer validator plugin settings.'
+  mapping:
+    min_similarity:
+      type: float
+      label: 'Minimum similarity for the answer to be considered valid.'
+
+# Similarity - Ranker answer validator plugin settings.
+ocha_ai.plugin.answer_validator.similarity_ranker:
+  type: mapping
+  label: 'Similarity - Ranker answer validator plugin settings.'
+  mapping:
+    min_similarity:
+      type: float
+      label: 'Minimum similarity for the answer to be considered valid.'
+
 # AWS Bedrock completion plugin settings.
 ocha_ai.plugin.completion.aws_bedrock:
   type: ocha_ai.plugin.completion
@@ -228,6 +287,15 @@ ocha_ai.plugin.embedding.azure_openai:
   type: ocha_ai.plugin.embedding
   label: 'AWS Bedrock embedding plugin settings.'
 
+# OCHA AI Helper ranker plugin settings
+ocha_ai.plugin.ranker.ocha_ai_helper_ranker:
+  type: ocha_ai.plugin.ranker
+  label: 'OCHA AI Helper ranker plugin settings.'
+  mapping:
+    endpoint:
+      type: string
+      label: 'Text ranking API endpoint.'
+
 # ReliefWeb source plugin settings.
 ocha_ai.plugin.source.reliefweb:
   type: ocha_ai.plugin.source
@@ -264,7 +332,7 @@ ocha_ai.plugin.text_extractor.mupdf:
 # Elasticsearch source plugin settings.
 ocha_ai.plugin.vector_store.elasticsearch:
   type: ocha_ai.plugin.vector_store
-  label: 'Elasticsearch source plugin settings.'
+  label: 'Elasticsearch vector store plugin settings.'
   mapping:
     url:
       type: string
@@ -294,10 +362,30 @@ ocha_ai.plugin.vector_store.elasticsearch:
       type: float
       label: 'Coefficient for the standard deviation to determine the similarity cut-off for relevancy'
 
+ocha_ai.plugin.vector_store.elasticsearch_flattened:
+  type: ocha_ai.plugin.vector_store.elasticsearch
+  label: 'Elasticsearch flattened vector store plugin settings.'
+  mapping:
+    expand_passage_before:
+      type: integer
+      label: 'Number of adjacent text passages to prepend to the text when passed as context.'
+    expand_passage_after:
+      type: integer
+      label: 'Number of adjacent text passages to append to the text when passed as context.'
+
+
+ocha_ai.plugin.text_splitter.nlp_sentence:
+  type: ocha_ai.plugin.text_splitter
+  label: 'NLP sentence text splitter settings.'
+  mapping:
+    endpoint:
+      type: string
+      label: 'NLP text splitting API endpoint.'
+
 ocha_ai.plugin.text_splitter.sentence:
   type: ocha_ai.plugin.text_splitter
-  label: 'Split a text in groups of sentences'
+  label: 'Sentence text splitter settings.'
 
 ocha_ai.plugin.text_splitter.token:
   type: ocha_ai.plugin.text_splitter
-  label: 'Split a text into passges based on their estimated token count'
+  label: 'Token text splitter settings.'

modules/ocha_ai_chat/config/install/ocha_ai_chat.settings.yml

@@ -7,7 +7,6 @@ defaults:
       format: 'plain_text'
     feedback: ''
     formatting: 'basic'
-    answer_min_similarity: 1.25
     answers:
       no_document: 'Sorry, no source documents were found.'
       no_passage: 'Sorry, I could not find information to answer the question.'
@@ -16,10 +15,15 @@ defaults:
       document_embedding_error: 'Sorry, there was an error trying to retrieve the documents to the answer to your question.'
       question_embedding_error: 'Sorry, there was an error trying to process the qestion.'
   plugins:
+    answer_validator:
+      plugin_id: similarity_embedding
     completion:
       plugin_id: aws_bedrock
     embedding:
       plugin_id: aws_bedrock
+    ranker:
+      plugin_id: NULL
+      limit: 5
     source:
       plugin_id: reliefweb
     text_extractor:

modules/ocha_ai_chat/config/schema/ocha_ai_chat.schema.yml

@@ -6,6 +6,12 @@ ocha_ai_chat.settings:
       type: mapping
       label: 'List of plugins.'
       mapping:
+        answer_validator:
+          type: sequence
+          label: 'List of answer validator plugins.'
+          sequence:
+            type: ocha_ai.plugin.answer_validator.[%key]
+            label: 'Settings for a answer validator plugin.'
         completion:
           type: sequence
           label: 'List of completion plugins.'
@@ -18,6 +24,12 @@ ocha_ai_chat.settings:
           sequence:
             type: ocha_ai.plugin.embedding.[%key]
             label: 'Settings for a embedding plugin.'
+        ranker:
+          type: sequence
+          label: 'List of ranker plugins.'
+          sequence:
+            type: ocha_ai.plugin.ranker.[%key]
+            label: 'Settings for a ranker plugin.'
         source:
           type: sequence
           label: 'List of source plugins.'
@@ -72,9 +84,6 @@ ocha_ai_chat.settings:
             formatting:
               type: string
               label: 'Formatting mode.'
-            answer_min_similarity:
-              type: float
-              label: 'Minimum similarity between the answer and the context passages to be considered valid.'
             answers:
               type: mapping
               mapping:
@@ -100,6 +109,13 @@ ocha_ai_chat.settings:
           type: mapping
           label: 'Default plugins.'
           mapping:
+            answer_validator:
+              type: mapping
+              label: 'Default answer validator plugin settings.'
+              mapping:
+                plugin_id:
+                  type: string
+                  label: 'Plugin ID.'
             completion:
               type: mapping
               label: 'Default completion plugin settings.'
@@ -114,6 +130,16 @@ ocha_ai_chat.settings:
                 plugin_id:
                   type: string
                   label: 'Plugin ID.'
+            ranker:
+              type: mapping
+              label: 'Default ranker plugin settings.'
+              mapping:
+                plugin_id:
+                  type: string
+                  label: 'Plugin ID.'
+                limit:
+                  type: integer
+                  label: 'Maximum number of relevant texts to return.'
             source:
               type: mapping
               label: 'Default source plugin settings.'

modules/ocha_ai_chat/ocha_ai_chat.services.yml

@@ -8,8 +8,10 @@ services:
       - '@current_user'
       - '@database'
       - '@datetime.time'
+      - '@plugin.manager.ocha_ai.answer_validator'
       - '@plugin.manager.ocha_ai.completion'
       - '@plugin.manager.ocha_ai.embedding'
+      - '@plugin.manager.ocha_ai.ranker'
       - '@plugin.manager.ocha_ai.source'
       - '@plugin.manager.ocha_ai.text_extractor'
       - '@plugin.manager.ocha_ai.text_splitter'