Merge branch 'main' into feat/queue-connector

.github/workflows/ci-test.yaml

@@ -0,0 +1,57 @@
+name: Run tox tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches: [main]
+
+jobs:
+  test:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+
+    - name: Cache tox environments
+      uses: actions/cache@v3
+      with:
+        path: .tox/
+        key: ${{ runner.os }}-tox-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }}
+        restore-keys: |
+          ${{ runner.os }}-tox-
+
+    - name: Install tox
+      run: pip install tox
+
+    - name: Run tox
+      id: tox
+      run: |
+        tox
+
+    - name: Render the report to the PR
+      uses: marocchino/sticky-pull-request-comment@v2
+      with:
+        header: worker-test-report
+        recreate: true
+        path: worker-report.md
+
+    - name: Output reports to the job summary when tests fail
+      shell: bash
+      run: |
+        if [ -f "worker-report.md" ]; then
+          echo "<details><summary>Worker Test Report</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          cat "worker-report.md" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+        fi

.gitignore

@@ -53,6 +53,7 @@ pip-delete-this-directory.txt
 
 # Unit test / coverage reports
 htmlcov/
+**/report.html
 .tox/
 .nox/
 .coverage

README.md

@@ -63,16 +63,17 @@ Unstract comes well documented. You can get introduced to the [basics of Unstrac
 
 ### LLM Providers
 
-|| Provider | Status |
-|---|---|---|
-|<img src="docs/assets/3rd_party/openai.png" width="32"/>| OpenAI | ✅ Working |
-|<img src="docs/assets/3rd_party/vertex_ai.png" width="32"/>| Google VertexAI, Gemini Pro | ✅ Working |
-|<img src="docs/assets/3rd_party/azure_openai.png" width="32"/>| Azure OpenAI | ✅ Working  |
-|<img src="docs/assets/3rd_party/palm.png" width="32"/>| Google PaLM | ✅ Working  |
-|<img src="docs/assets/3rd_party/anyscale.png" width="32"/>| Anyscale | ✅ Working |
-|<img src="docs/assets/3rd_party/mistral_ai.png" width="32"/>| Mistral AI | ✅ Working |
-|<img src="docs/assets/3rd_party/anthropic.png" width="32"/>| Anthropic | ✅ Working |
-|<img src="docs/assets/3rd_party/replicate.png" width="32"/>| Replicate | 🗓️ Coming soon! |
+|| Provider                                                       | Status                      |
+|----------------------------------------------------------------|-----------------------------|---|
+| <img src="docs/assets/3rd_party/openai.png" width="32"/>       | OpenAI                      | ✅ Working |
+| <img src="docs/assets/3rd_party/vertex_ai.png" width="32"/>    | Google VertexAI, Gemini Pro | ✅ Working |
+| <img src="docs/assets/3rd_party/azure_openai.png" width="32"/> | Azure OpenAI                | ✅ Working  |
+| <img src="docs/assets/3rd_party/palm.png" width="32"/>         | Google PaLM                 | ✅ Working  |
+| <img src="docs/assets/3rd_party/anyscale.png" width="32"/>     | Anyscale                    | ✅ Working |
+| <img src="docs/assets/3rd_party/mistral_ai.png" width="32"/>   | Mistral AI                  | ✅ Working |
+| <img src="docs/assets/3rd_party/anthropic.png" width="32"/>    | Anthropic                   | ✅ Working |
+| <img src="docs/assets/3rd_party/ollama.png" width="32"/>       | Ollama                      | ✅ Working |
+| <img src="docs/assets/3rd_party/replicate.png" width="32"/>    | Replicate                   | 🗓️ Coming soon! |
 
 
 ### Vector Databases

backend/adapter_processor/models.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import uuid
 from typing import Any
 
@@ -9,13 +10,15 @@
 from django.db.models import QuerySet
 from unstract.adapters.adapterkit import Adapterkit
 from unstract.adapters.enums import AdapterTypes
-from unstract.adapters.llm.llm_adapter import LLMAdapter
+from unstract.adapters.exceptions import AdapterError
 from utils.models.base_model import BaseModel
 
 ADAPTER_NAME_SIZE = 128
 VERSION_NAME_SIZE = 64
 ADAPTER_ID_LENGTH = 128
 
+logger = logging.getLogger(__name__)
+
 
 class AdapterInstanceModelManager(models.Manager):
     def get_queryset(self) -> QuerySet[Any]:
@@ -146,9 +149,11 @@ def get_context_window_size(self) -> int:
         adapter_metadata = self.get_adapter_meta_data()
         # Get the adapter_instance
         adapter_class = Adapterkit().get_adapter_class_by_adapter_id(self.adapter_id)
-        adapter_instance = adapter_class(adapter_metadata)
-        if isinstance(adapter_instance, LLMAdapter):
+        try:
+            adapter_instance = adapter_class(adapter_metadata)
             return adapter_instance.get_context_window_size()
+        except AdapterError as e:
+            logger.warning(f"Unable to retrieve context window size - {e}")
         return 0
 
 

backend/adapter_processor/serializers.py

@@ -9,6 +9,7 @@
 from rest_framework import serializers
 from rest_framework.serializers import ModelSerializer
 from unstract.adapters.constants import Common as common
+from unstract.adapters.enums import AdapterTypes
 
 from backend.constants import FieldLengthConstants as FLC
 from backend.serializers import AuditSerializer
@@ -63,9 +64,12 @@ def to_representation(self, instance: AdapterInstance) -> dict[str, str]:
 
         adapter_metadata = instance.get_adapter_meta_data()
         rep[AdapterKeys.ADAPTER_METADATA] = adapter_metadata
-        adapter_metadata[AdapterKeys.ADAPTER_CONTEXT_WINDOW_SIZE] = (
-            instance.get_context_window_size()
-        )
+        # Retrieve context window if adapter is a LLM
+        # For other adapter types, context_window is not relevant.
+        if instance.adapter_type == AdapterTypes.LLM.value:
+            adapter_metadata[AdapterKeys.ADAPTER_CONTEXT_WINDOW_SIZE] = (
+                instance.get_context_window_size()
+            )
 
         rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(
             instance.adapter_id, common.ICON