SDK and Adapters version bump

backend/sample.env

@@ -89,9 +89,9 @@ PROMPT_PORT=3003
 PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
 
 # Structure Tool
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.26"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.27"
 STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.26"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.27"
 
 # Feature Flags
 EVALUATION_SERVER_IP=localhost

backend/usage/migrations/0003_usage_cost_in_dollars_usage_llm_usage_reason_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.1 on 2024-06-25 04:29
+# Generated by Django 4.2.1 on 2024-06-26 01:29
 
 from django.db import migrations, models
 
@@ -23,7 +23,12 @@ class Migration(migrations.Migration):
             name="llm_usage_reason",
             field=models.CharField(
                 blank=True,
-                db_comment="Reason for LLM usage, e.g., 'extraction', 'challenge' or 'summarize'. Empty if usage_type is 'embedding'.",
+                choices=[
+                    ("extraction", "extraction"),
+                    ("challenge", "challenge"),
+                    ("summarize", "summarize"),
+                ],
+                db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",
                 max_length=255,
                 null=True,
             ),
@@ -32,7 +37,13 @@ class Migration(migrations.Migration):
             model_name="usage",
             name="usage_type",
             field=models.CharField(
-                db_comment="Type of usage, either 'llm' or 'embedding'", max_length=255
+                choices=[("llm", "llm"), ("embedding", "embedding")],
+                db_comment="Type of usage, either 'llm' or 'embedding'",
+                max_length=255,
             ),
         ),
+        migrations.AddIndex(
+            model_name="usage",
+            index=models.Index(fields=["run_id"], name="token_usage_run_id_cd3578_idx"),
+        ),
     ]

backend/usage/models.py

@@ -27,16 +27,23 @@ class Usage(BaseModel):
         max_length=255, null=True, blank=True, db_comment="Identifier for the run"
     )
     usage_type = models.CharField(
-        max_length=255, db_comment="Type of usage, either 'llm' or 'embedding'"
+        max_length=255,
+        choices=[
+            ("llm", "llm"),
+            ("embedding", "embedding"),
+        ],
+        db_comment="Type of usage, either 'llm' or 'embedding'",
     )
     llm_usage_reason = models.CharField(
         max_length=255,
+        choices=[
+            ("extraction", "extraction"),
+            ("challenge", "challenge"),
+            ("summarize", "summarize"),
+        ],
         null=True,
         blank=True,
-        db_comment=(
-            "Reason for LLM usage, e.g., 'extraction', 'challenge' or 'summarize'. "
-            "Empty if usage_type is 'embedding'."
-        ),
+        db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",
     )
     model_name = models.CharField(max_length=255, db_comment="Name of the model used")
     embedding_tokens = models.IntegerField(
@@ -56,3 +63,6 @@ def __str__(self):
 
     class Meta:
         db_table = "token_usage"
+        indexes = [
+            models.Index(fields=["run_id"]),
+        ]

platform-service/src/unstract/platform_service/helper.py

@@ -1,12 +1,12 @@
 import json
 import os
 from datetime import datetime, timedelta
+from logging import Logger
 from typing import Any
 
 import numpy
 import peewee
 import requests
-from flask import current_app
 from unstract.platform_service.exceptions import CustomException
 
 
@@ -83,11 +83,12 @@ def __init__(
         url: str,
         ttl_days: int,
         file_path: str,
+        logger: Logger,
     ):
         self.ttl_days = ttl_days
         self.url = url
         self.file_path = file_path
-        self.logger = current_app.logger
+        self.logger = logger
 
     def calculate_cost(self, model_name, input_tokens, output_tokens):
         cost = 0.0

platform-service/src/unstract/platform_service/main.py

@@ -63,6 +63,7 @@
     url=MODEL_PRICES_URL,
     ttl_days=MODEL_PRICES_TTL_IN_DAYS,
     file_path=MODEL_PRICES_FILE_PATH,
+    logger=app.logger,
 )
 
 

prompt-service/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "flask~=3.0",
     "llama-index==0.10.38",
     "python-dotenv==1.0.0",
-    "unstract-sdk~=0.33.0",
+    "unstract-sdk~=0.33.1",
     "redis>=5.0.3",
     "unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",
 ]

pyproject.toml

@@ -40,8 +40,8 @@ hook-check-django-migrations = [
     "psycopg2-binary==2.9.9",
     "python-dotenv==1.0.0",
     "python-magic==0.4.27",
-    "unstract-sdk~=0.33.0",
-    "unstract-adapters~=0.19.1",
+    "unstract-sdk~=0.33.1",
+    "unstract-adapters~=0.19.2",
     "-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
     "-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",
     "-e unstract-flags @ file:///${PROJECT_ROOT}/unstract/flags",

tools/classifier/requirements.txt

@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.33.0
+unstract-sdk~=0.33.1

tools/structure/requirements.txt

@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.33.0
+unstract-sdk~=0.33.1

tools/text_extractor/requirements.txt

@@ -1,4 +1,4 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.33.0
+unstract-sdk~=0.33.1

unstract/tool-registry/pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "docker~=6.1.3",
     "jsonschema~=4.18.2",
     "PyYAML~=6.0.1",
-    "unstract-adapters~=0.19.1",
+    "unstract-adapters~=0.19.2",
     # ! IMPORTANT!
     # Local dependencies usually need to be added as:
     # https://pdm-project.org/latest/usage/dependency/#local-dependencies