diff --git a/.gitignore b/.gitignore index 9128508d7..a54e84f10 100644 --- a/.gitignore +++ b/.gitignore @@ -134,8 +134,8 @@ celerybeat.pid # Environments test*.env -*.env -!sample.env +.env +.*.env .env.export .venv* env/ @@ -647,7 +647,8 @@ tools/*/data_dir/ backend/backend/settings/* !backend/backend/settings/__init__.py !backend/backend/settings/base.py -!backend/backend/settings/dev.py +!backend/backend/settings/platform.py +!backend/backend/settings/celery.py !backend/backend/settings/test.py # Local Dependencies for docker testing diff --git a/backend/README.md b/backend/README.md index 4cb24a646..f53b299c5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -18,7 +18,7 @@ Contains the backend services for Unstract written with Django and DRF. - Copy `sample.env` into `.env` and update the necessary variables. For eg: ``` -DJANGO_SETTINGS_MODULE='backend.settings.dev' +DJANGO_SETTINGS_MODULE='backend.settings.platform' DB_HOST='localhost' DB_USER='unstract_dev' DB_PASSWORD='unstract_pass' diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.py index f8b41e921..f1160c14c 100644 --- a/backend/account/authentication_controller.py +++ b/backend/account/authentication_controller.py @@ -113,7 +113,8 @@ def user_organizations(self, request: Request) -> Any: try: organizations = self.auth_service.user_organizations(request) except Exception as ex: - # + Logger.error(f"Failed to get user orgs: {ex}") + self.user_logout(request) response = Response( diff --git a/backend/account/authentication_plugin_registry.py b/backend/account/authentication_plugin_registry.py index cd630fdaf..c3d7dbc04 100644 --- a/backend/account/authentication_plugin_registry.py +++ b/backend/account/authentication_plugin_registry.py @@ -43,14 +43,11 @@ def _load_plugins() -> dict[str, dict[str, Any]]: PluginConfig.AUTH_METADATA: module.metadata, } Logger.info( - "Loaded auth plugin: %s, is_active: %s", - module.metadata["name"], - module.metadata["is_active"], + "Loaded active authentication plugin: %s", module.metadata["name"] ) else: - Logger.warning( - "Metadata is not active for %s authentication module.", - auth_module_name, + Logger.info( + "Skipping inactive authentication plugin: %s", auth_module_name ) except ModuleNotFoundError as exception: Logger.error( diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py index e921073a8..e69de29bb 100644 --- a/backend/backend/__init__.py +++ b/backend/backend/__init__.py @@ -1,3 +0,0 @@ -from .celery_service import app as celery_app - -__all__ = ["celery_app"] diff --git a/backend/backend/asgi.py b/backend/backend/asgi.py index 07774528a..b9687bca8 100644 --- a/backend/backend/asgi.py +++ b/backend/backend/asgi.py @@ -8,13 +8,17 @@ import os -from django.core.asgi import get_asgi_application -from dotenv import load_dotenv +from dotenv import find_dotenv, load_dotenv + +load_dotenv(find_dotenv() or "") load_dotenv() + os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) +from django.core.asgi import get_asgi_application # noqa: E402 + application = get_asgi_application() diff --git a/backend/backend/celery_service.py b/backend/backend/celery.py similarity index 66% rename from backend/backend/celery_service.py rename to backend/backend/celery.py index 99effcd78..4580a1908 100644 --- a/backend/backend/celery_service.py +++ b/backend/backend/celery.py @@ -2,20 +2,25 @@ import os -from celery import Celery -from django.conf import settings -from utils.constants import ExecutionLogConstants +from dotenv import find_dotenv, load_dotenv -from backend.celery_task import TaskRegistry - -# Set the default Django settings module for the 'celery' program. +# NOTE: +# Do this before loading any environment variables. os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.celery"), ) +# Load environment variables. +load_dotenv(find_dotenv() or "") + +from celery import Celery # noqa: E402 +from django.conf import settings # noqa: E402 +from utils.celery.constants import ExecutionLogConstants # noqa: E402 +from utils.celery.task_registry import TaskRegistry # noqa: E402 + # Create a Celery instance. Default time zone is UTC. -app = Celery("backend") +app = Celery("celery") # To register the custom tasks. TaskRegistry() @@ -24,7 +29,6 @@ app.conf.broker_url = settings.CELERY_BROKER_URL app.conf.result_backend = settings.CELERY_RESULT_BACKEND - # Load task modules from all registered Django app configs. app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/backend/backend/flowerconfig.py b/backend/backend/flowerconfig.py deleted file mode 100644 index 1d25d6458..000000000 --- a/backend/backend/flowerconfig.py +++ /dev/null @@ -1,15 +0,0 @@ -# Flower is a real-time web based monitor and administration tool -# for Celery. It’s under active development, -# but is already an essential tool. -from django.conf import settings - -# Broker URL -BROKER_URL = settings.CELERY_BROKER_URL - -# Flower web port -PORT = 5555 - -# Enable basic authentication (when required) -# basic_auth = { -# 'username': 'password' -# } diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 1ebb46bc0..67d1cb7c3 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -12,7 +12,6 @@ import os from pathlib import Path from typing import Optional -from urllib.parse import quote_plus from dotenv import find_dotenv, load_dotenv from utils.common_utils import CommonUtils @@ -131,10 +130,6 @@ def get_required_setting( LOG_HISTORY_CONSUMER_INTERVAL = int( get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60") ) -LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30")) -CELERY_BROKER_URL = get_required_setting( - "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}" -) INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL")) NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5")) @@ -367,22 +362,10 @@ def get_required_setting( } -# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1" -# Postgres as result backend -CELERY_RESULT_BACKEND = ( - f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}" - f"@{DB_HOST}:{DB_PORT}/{DB_NAME}" -) -CELERY_ACCEPT_CONTENT = ["json"] -CELERY_TASK_SERIALIZER = "json" -CELERY_RESULT_SERIALIZER = "json" -CELERY_TIMEZONE = "UTC" -CELERY_TASK_MAX_RETRIES = 3 -CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task - # Feature Flag FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"} + # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators diff --git a/backend/backend/settings/celery.py b/backend/backend/settings/celery.py new file mode 100644 index 000000000..147a93838 --- /dev/null +++ b/backend/backend/settings/celery.py @@ -0,0 +1,100 @@ +import os +from typing import Any, Optional +from urllib.parse import quote_plus + +from backend.constants import FeatureFlag + +# Requires PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env setting +# to be loaded prior. +from unstract.flags.feature_flag import check_feature_flag_status + +missing_settings = [] + + +def get_required_setting( + setting_key: str, default: Optional[Any] = None +) -> Optional[str]: + """Get the value of an environment variable specified by the given key. Add + missing keys to `missing_settings` so that exception can be raised at the + end. + + Args: + key (str): The key of the environment variable + default (Optional[str], optional): Default value to return incase of + env not found. Defaults to None. + + Returns: + Optional[str]: The value of the environment variable if found, + otherwise the default value. + """ + data = os.environ.get(setting_key, default) + if not data: + missing_settings.append(setting_key) + return data + + +DB_NAME = os.environ.get("DB_NAME", "unstract_db") +DB_USER = os.environ.get("DB_USER", "unstract_dev") +DB_HOST = os.environ.get("DB_HOST", "backend-db-1") +DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass") +DB_PORT = os.environ.get("DB_PORT", 5432) +if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2): + DB_ENGINE = "django.db.backends.postgresql" +else: + DB_ENGINE = "django_tenants.postgresql_backend" + + +REDIS_USER = os.environ.get("REDIS_USER", "default") +REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "") +REDIS_HOST = os.environ.get("REDIS_HOST", "unstract-redis") +REDIS_PORT = os.environ.get("REDIS_PORT", "6379") +REDIS_DB = os.environ.get("REDIS_DB", "") + +ENABLE_LOG_HISTORY = get_required_setting("ENABLE_LOG_HISTORY") +LOG_HISTORY_CONSUMER_INTERVAL = int( + get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60") +) +LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30")) + +CORS_ALLOWED_ORIGINS = [ + "http://localhost:3000", + "http://127.0.0.1:3000", + "http://frontend.unstract.localhost", + # Other allowed origins if needed +] + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases +DATABASES = { + "default": { + "ENGINE": DB_ENGINE, + "NAME": f"{DB_NAME}", + "USER": f"{DB_USER}", + "HOST": f"{DB_HOST}", + "PASSWORD": f"{DB_PASSWORD}", + "PORT": f"{DB_PORT}", + "ATOMIC_REQUESTS": True, + } +} + +# SocketIO connection manager +SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}" + +CELERY_BROKER_URL = get_required_setting( + "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}" +) +# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1" +# Postgres as result backend +CELERY_RESULT_BACKEND = ( + f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}" + f"@{DB_HOST}:{DB_PORT}/{DB_NAME}" +) +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +CELERY_TIMEZONE = "UTC" +CELERY_TASK_MAX_RETRIES = 3 +CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task + + +INSTALLED_APPS = ["django.contrib.contenttypes", "django_celery_beat"] diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/platform.py similarity index 97% rename from backend/backend/settings/dev.py rename to backend/backend/settings/platform.py index 8765cd234..f87099c04 100644 --- a/backend/backend/settings/dev.py +++ b/backend/backend/settings/platform.py @@ -1,6 +1,6 @@ from backend.settings.base import * # noqa: F401, F403 -DEBUG = True +DEBUG = False X_FRAME_OPTIONS = "http://localhost:3000" X_FRAME_OPTIONS = "ALLOW-FROM http://localhost:3000" diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py index 9a654eb50..b00212a83 100644 --- a/backend/backend/wsgi.py +++ b/backend/backend/wsgi.py @@ -8,18 +8,19 @@ import os -from django.conf import settings -from django.core.wsgi import get_wsgi_application -from dotenv import load_dotenv -from utils.log_events import start_server +from dotenv import find_dotenv, load_dotenv -load_dotenv() +load_dotenv(find_dotenv() or "") os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) +from django.conf import settings # noqa: E402 +from django.core.wsgi import get_wsgi_application # noqa: E402 +from utils.log_events import start_server # noqa: E402 + django_app = get_wsgi_application() application = start_server(django_app, f"{settings.PATH_PREFIX}/socket") diff --git a/backend/manage.py b/backend/manage.py index c5d2f5890..bb3de315f 100755 --- a/backend/manage.py +++ b/backend/manage.py @@ -12,7 +12,7 @@ def main() -> None: load_dotenv() os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) try: from django.core.management import execute_from_command_line diff --git a/backend/pdm.lock b/backend/pdm.lock index 7facc39a7..eba0eec4b 100644 --- a/backend/pdm.lock +++ b/backend/pdm.lock @@ -182,6 +182,9 @@ version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["default", "dev"] +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.9\"", +] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -195,6 +198,7 @@ summary = "The official Python library for the anthropic API" groups = ["default", "dev"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -266,6 +270,9 @@ requires_python = ">=3.7" summary = "Timeout context manager for asyncio programs" groups = ["default", "dev"] marker = "python_version < \"3.12.0\"" +dependencies = [ + "typing-extensions>=3.6.5; python_version < \"3.8\"", +] files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -314,6 +321,9 @@ version = "24.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["default", "dev"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -337,6 +347,9 @@ name = "azure-common" version = "1.1.28" summary = "Microsoft Azure Client Library for Python (Common)" groups = ["default"] +dependencies = [ + "azure-nspkg; python_version < \"3.0\"", +] files = [ {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, @@ -364,8 +377,11 @@ version = "0.0.53" summary = "Azure Data Lake Store Filesystem Client Library for Python" groups = ["default", "dev"] dependencies = [ + "azure-nspkg; python_version < \"3.0\"", "cffi", + "futures; python_version <= \"2.7\"", "msal<2,>=1.16.0", + "pathlib2; python_version < \"3.4\"", "requests>=2.20.0", ] files = [ @@ -608,11 +624,13 @@ requires_python = ">=3.8" summary = "Distributed Task Queue." groups = ["default"] dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "billiard<5.0,>=4.2.0", "click-didyoumean>=0.3.0", "click-plugins>=1.1.1", "click-repl>=0.2.0", "click<9.0,>=8.1.2", + "importlib-metadata>=3.6; python_version < \"3.8\"", "kombu<6.0,>=5.3.4", "python-dateutil>=2.8.2", "tzdata>=2022.7", @@ -747,6 +765,7 @@ summary = "Composable command line interface toolkit" groups = ["default", "dev"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -855,10 +874,6 @@ files = [ {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, @@ -946,6 +961,7 @@ summary = "A high-level Python web framework that encourages rapid development a groups = ["default"] dependencies = [ "asgiref<4,>=3.6.0", + "backports-zoneinfo; python_version < \"3.9\"", "sqlparse>=0.3.1", "tzdata; sys_platform == \"win32\"", ] @@ -961,9 +977,11 @@ summary = "Database-backed Periodic Tasks." groups = ["default"] dependencies = [ "Django<5.0,>=2.2", + "backports-zoneinfo; python_version < \"3.9\"", "celery<6.0,>=5.2.3", "cron-descriptor>=1.2.32", "django-timezone-field>=5.0", + "importlib-metadata<5.0; python_version < \"3.8\"", "python-crontab>=2.3.4", "tzdata", ] @@ -1035,6 +1053,7 @@ summary = "A Django app providing DB, form, and REST framework fields for zonein groups = ["default"] dependencies = [ "Django<6.0,>=3.2", + "backports-zoneinfo<0.3.0,>=0.2.1; python_version < \"3.9\"", ] files = [ {file = "django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb"}, @@ -1503,6 +1522,7 @@ groups = ["default", "dev"] dependencies = [ "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.6", "google-auth<3.0dev,>=1.25.0", + "importlib-metadata>1.0.0; python_version < \"3.8\"", ] files = [ {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, @@ -1828,6 +1848,7 @@ requires_python = ">=3.7" summary = "WSGI HTTP Server for UNIX" groups = ["deploy"] dependencies = [ + "importlib-metadata; python_version < \"3.8\"", "packaging", ] files = [ @@ -1841,6 +1862,9 @@ version = "0.14.0" requires_python = ">=3.7" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" groups = ["default", "dev"] +dependencies = [ + "typing-extensions; python_version < \"3.8\"", +] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1895,6 +1919,7 @@ summary = "A comprehensive HTTP client library." groups = ["default", "dev"] dependencies = [ "pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2; python_version > \"3.0\"", + "pyparsing<3,>=2.4.2; python_version < \"3.0\"", ] files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -2097,7 +2122,9 @@ summary = "An implementation of JSON Schema validation for Python" groups = ["default", "dev"] dependencies = [ "attrs>=22.2.0", + "importlib-resources>=1.4.0; python_version < \"3.9\"", "jsonschema-specifications>=2023.03.6", + "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"", "referencing>=0.28.4", "rpds-py>=0.7.1", ] @@ -2113,6 +2140,7 @@ requires_python = ">=3.9" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" groups = ["default", "dev"] dependencies = [ + "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", ] files = [ @@ -2128,6 +2156,7 @@ summary = "Messaging library for Python." groups = ["default", "dev"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", "typing-extensions; python_version < \"3.10\"", "vine", ] @@ -2964,6 +2993,7 @@ summary = "The official Python library for the openai API" groups = ["default", "dev"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -3037,6 +3067,7 @@ groups = ["default", "dev"] dependencies = [ "numpy<2,>=1.22.4; python_version < \"3.11\"", "numpy<2,>=1.23.2; python_version == \"3.11\"", + "numpy<2,>=1.26.0; python_version >= \"3.12\"", "python-dateutil>=2.8.2", "pytz>=2020.1", "tzdata>=2022.1", @@ -3088,6 +3119,8 @@ groups = ["default", "dev"] dependencies = [ "charset-normalizer>=2.0.0", "cryptography>=36.0.0", + "importlib-metadata; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "pdfminer.six-20231228-py3-none-any.whl", hash = "sha256:e8d3c3310e6fbc1fe414090123ab01351634b4ecb021232206c4c9a8ca3e3b8f"}, @@ -3188,6 +3221,7 @@ dependencies = [ "tqdm>=4.64.1", "typing-extensions>=3.7.4", "urllib3>=1.26.0; python_version >= \"3.8\" and python_version < \"3.12\"", + "urllib3>=1.26.5; python_version ~= \"3.12\"", ] files = [ {file = "pinecone_client-3.2.2-py3-none-any.whl", hash = "sha256:7e492fdda23c73726bc0cb94c689bb950d06fb94e82b701a0c610c2e830db327"}, @@ -3200,6 +3234,9 @@ version = "3.11.0" requires_python = ">=3.7" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." groups = ["default", "dev"] +dependencies = [ + "typing-extensions>=4.7.1; python_version < \"3.8\"", +] files = [ {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, @@ -3626,8 +3663,10 @@ dependencies = [ "environs<=9.5.0", "grpcio>=1.49.1", "milvus-lite<2.5.0,>=2.4.0; sys_platform != \"win32\"", + "numpy<1.25.0; python_version <= \"3.8\"", "pandas>=1.2.4", "protobuf>=3.20.0", + "setuptools<70.1; python_version <= \"3.8\"", "setuptools>69", "ujson>=2.0.0", ] @@ -3726,6 +3765,7 @@ requires_python = ">=3.6" summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" groups = ["default", "dev"] dependencies = [ + "dataclasses; python_version < \"3.7\"", "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ @@ -3953,7 +3993,9 @@ dependencies = [ "grpcio-tools>=1.41.0", "grpcio>=1.41.0", "httpx[http2]>=0.20.0", + "numpy<1.21; python_version < \"3.8\"", "numpy>=1.21; python_version >= \"3.8\" and python_version < \"3.12\"", + "numpy>=1.26; python_version >= \"3.12\"", "portalocker<3.0.0,>=2.7.0", "pydantic>=1.10.8", "urllib3<3,>=1.26.14", @@ -3971,6 +4013,8 @@ summary = "Python client for Redis database and key-value store" groups = ["default", "dev"] dependencies = [ "async-timeout>=4.0.3; python_full_version < \"3.11.3\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, @@ -4402,6 +4446,7 @@ dependencies = [ "cryptography<42.0.0,>=3.1.0", "filelock<4,>=3.5", "idna<4,>=2.5", + "importlib-metadata; python_version < \"3.8\"", "packaging", "platformdirs<4.0.0,>=2.6.0", "pyOpenSSL<24.0.0,>=16.2.0", @@ -4527,6 +4572,7 @@ summary = "Database Abstraction Library" groups = ["default", "dev"] dependencies = [ "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"", + "importlib-metadata; python_version < \"3.8\"", "typing-extensions>=4.6.0", ] files = [ @@ -4732,19 +4778,6 @@ files = [ {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, @@ -4854,6 +4887,7 @@ groups = ["default", "dev"] dependencies = [ "mypy-extensions>=0.3.0", "typing-extensions>=3.7.4", + "typing>=3.7.4; python_version < \"3.5\"", ] files = [ {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, @@ -4914,11 +4948,6 @@ files = [ {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbd8fd427f57a03cff3ad6574b5e299131585d9727c8c366da4624a9069ed746"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beeaf1c48e32f07d8820c705ff8e645f8afa690cca1544adba4ebfa067efdc88"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:baed37ea46d756aca2955e99525cc02d9181de67f25515c468856c38d52b5f3b"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7663960f08cd5a2bb152f5ee3992e1af7690a64c0e26d31ba7b3ff5b2ee66337"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d8640fb4072d36b08e95a3a380ba65779d356b2fee8696afeb7794cf0902d0a1"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78778a3aa7aafb11e7ddca4e29f46bc5139131037ad628cc10936764282d6753"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0111b27f2d5c820e7f2dbad7d48e3338c824e7ac4d2a12da3dc6061cc39c8e6"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c66962ca7565605b355a9ed478292da628b8f18c0f2793021ca4425abf8b01e5"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba43cc34cce49cf2d4bc76401a754a81202d8aa926d0e2b79f0ee258cb15d3a4"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac56eb983edce27e7f51d05bc8dd820586c6e6be1c5216a6809b0c668bb312b8"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44bd4b23a0e723bf8b10628288c2c7c335161d6840013d4d5de20e48551773b"}, @@ -5123,6 +5152,9 @@ name = "wcwidth" version = "0.2.13" summary = "Measures the displayed width of unicode strings in a terminal" groups = ["default"] +dependencies = [ + "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"", +] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, diff --git a/backend/sample.celery.env b/backend/sample.celery.env new file mode 100644 index 000000000..832c9c3a3 --- /dev/null +++ b/backend/sample.celery.env @@ -0,0 +1,30 @@ +DJANGO_SETTINGS_MODULE='backend.settings.celery' + +# Default log level +DEFAULT_LOG_LEVEL="INFO" + +# Postgres DB envs +DB_HOST='unstract-db' +DB_USER='unstract_dev' +DB_PASSWORD='unstract_pass' +DB_NAME='unstract_db' +DB_PORT=5432 + +# Redis +REDIS_HOST="unstract-redis" +REDIS_PORT=6379 +REDIS_PASSWORD="" +REDIS_USER=default + +# Protocol buffers generated code. +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Enable logging of workflow history. +ENABLE_LOG_HISTORY=True +# Interval in seconds for periodic consumer operations. +LOG_HISTORY_CONSUMER_INTERVAL=30 +# Maximum number of logs to insert in a single batch. +LOGS_BATCH_LIMIT=30 + +# Celery Configuration +CELERY_BROKER_URL="redis://unstract-redis:6379" diff --git a/backend/sample.env b/backend/sample.env index e48c508e6..d7371501b 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -1,4 +1,4 @@ -DJANGO_SETTINGS_MODULE='backend.settings.dev' +DJANGO_SETTINGS_MODULE='backend.settings.platform' # NOTE: Change below to True if you are running in HTTPS mode. SESSION_COOKIE_SECURE=False @@ -88,6 +88,8 @@ STRUCTURE_TOOL_IMAGE_TAG="0.0.47" # Feature Flags EVALUATION_SERVER_IP=unstract-flipt EVALUATION_SERVER_PORT=9000 +EVALUATION_SERVER_WARNINGS="false" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python diff --git a/backend/utils/celery/constants.py b/backend/utils/celery/constants.py new file mode 100644 index 000000000..8562bcee9 --- /dev/null +++ b/backend/utils/celery/constants.py @@ -0,0 +1,19 @@ +from django.conf import settings +from utils.common_utils import CommonUtils + + +class ExecutionLogConstants: + """Constants for ExecutionLog. + + Attributes: + IS_ENABLED (bool): Whether to enable log history. + LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch. + LOG_QUEUE_NAME (str): The name of the queue to store log history. + CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log + history consumers. + """ + + IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY) + LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT + LOG_QUEUE_NAME: str = "log_history_queue" + CELERY_QUEUE_NAME = "celery_periodic_logs" diff --git a/backend/backend/celery_task.py b/backend/utils/celery/task_registry.py similarity index 100% rename from backend/backend/celery_task.py rename to backend/utils/celery/task_registry.py diff --git a/backend/utils/constants.py b/backend/utils/constants.py index a92f397a4..fb21fa56f 100644 --- a/backend/utils/constants.py +++ b/backend/utils/constants.py @@ -1,13 +1,6 @@ -import os - from django.conf import settings from utils.common_utils import CommonUtils -os.environ.setdefault( - "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), -) - class Account: CREATED_BY = "created_by" @@ -61,7 +54,6 @@ class ExecutionLogConstants: CONSUMER_INTERVAL (int): The interval (in seconds) between log history consumers. LOG_QUEUE_NAME (str): The name of the queue to store log history. - LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch. CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log history consumers. PERIODIC_TASK_NAME (str): The name of the Celery periodic task to schedule @@ -71,7 +63,6 @@ class ExecutionLogConstants: IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY) CONSUMER_INTERVAL: int = settings.LOG_HISTORY_CONSUMER_INTERVAL - LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT LOG_QUEUE_NAME: str = "log_history_queue" CELERY_QUEUE_NAME = "celery_periodic_logs" PERIODIC_TASK_NAME = "workflow_log_history" diff --git a/dev-env-cli.sh b/dev-env-cli.sh index a0826c159..c9f14dc1b 100755 --- a/dev-env-cli.sh +++ b/dev-env-cli.sh @@ -167,6 +167,14 @@ parse_args() { debug "OPTION verbose: $opt_verbose" } +check_for_active_venv() { + if [[ -v VIRTUAL_ENV ]] && [[ -n "$VIRTUAL_ENV" ]]; then + echo -e "$red_text""Detected active virtual env from: $VIRTUAL_ENV.""$default_text" + echo -e "$red_text""Please run 'deactivate' first. Exiting.""$default_text" + exit 1 + fi +} + setup_venv() { if [ "$opt_setup_venv" = false ]; then return @@ -288,6 +296,7 @@ run_pre_commit_hook() { # Run Unstract platform - BEGIN # check_dependencies +check_for_active_venv opt_setup_venv=false opt_activate_venv=false diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 2c4359990..dcf17fad1 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -42,7 +42,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend worker --loglevel=info -Q celery --autoscale=${WORKER_AUTOSCALE}" env_file: - - ../backend/.env + - ../backend/.celery.env depends_on: - redis environment: @@ -100,7 +100,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend flower --port=5555 --purge_offline_workers=5" env_file: - - ../backend/.env + - ../backend/.celery.env depends_on: - worker - worker-logging @@ -126,7 +126,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l INFO" env_file: - - ../backend/.env + - ../backend/.celery.env - ./essentials.env depends_on: - db diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock index a9803ba44..048eaefb7 100644 --- a/prompt-service/pdm.lock +++ b/prompt-service/pdm.lock @@ -116,6 +116,9 @@ version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["default"] +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.9\"", +] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -129,6 +132,7 @@ summary = "The official Python library for the anthropic API" groups = ["default"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -166,6 +170,9 @@ requires_python = ">=3.7" summary = "Timeout context manager for asyncio programs" groups = ["default"] marker = "python_version < \"3.12.0\"" +dependencies = [ + "typing-extensions>=3.6.5; python_version < \"3.8\"", +] files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -214,6 +221,9 @@ version = "24.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["default"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -461,6 +471,7 @@ summary = "Composable command line interface toolkit" groups = ["default"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -835,6 +846,7 @@ groups = ["default"] dependencies = [ "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.6", "google-auth<3.0dev,>=1.25.0", + "importlib-metadata>1.0.0; python_version < \"3.8\"", ] files = [ {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, @@ -1143,6 +1155,7 @@ requires_python = ">=3.7" summary = "WSGI HTTP Server for UNIX" groups = ["deploy"] dependencies = [ + "importlib-metadata; python_version < \"3.8\"", "packaging", ] files = [ @@ -1156,6 +1169,9 @@ version = "0.14.0" requires_python = ">=3.7" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" groups = ["default"] +dependencies = [ + "typing-extensions; python_version < \"3.8\"", +] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1394,7 +1410,9 @@ summary = "An implementation of JSON Schema validation for Python" groups = ["default"] dependencies = [ "attrs>=22.2.0", + "importlib-resources>=1.4.0; python_version < \"3.9\"", "jsonschema-specifications>=2023.03.6", + "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"", "referencing>=0.28.4", "rpds-py>=0.7.1", ] @@ -1410,6 +1428,7 @@ requires_python = ">=3.9" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" groups = ["default"] dependencies = [ + "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", ] files = [ @@ -1425,6 +1444,7 @@ summary = "Messaging library for Python." groups = ["default"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", "typing-extensions; python_version < \"3.10\"", "vine", ] @@ -2255,6 +2275,7 @@ summary = "The official Python library for the openai API" groups = ["default"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -2328,6 +2349,7 @@ groups = ["default"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", + "numpy>=1.26.0; python_version >= \"3.12\"", "python-dateutil>=2.8.2", "pytz>=2020.1", "tzdata>=2022.7", @@ -2366,6 +2388,8 @@ groups = ["default"] dependencies = [ "charset-normalizer>=2.0.0", "cryptography>=36.0.0", + "importlib-metadata; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "pdfminer.six-20231228-py3-none-any.whl", hash = "sha256:e8d3c3310e6fbc1fe414090123ab01351634b4ecb021232206c4c9a8ca3e3b8f"}, @@ -2475,6 +2499,7 @@ dependencies = [ "tqdm>=4.64.1", "typing-extensions>=3.7.4", "urllib3>=1.26.0; python_version >= \"3.8\" and python_version < \"3.12\"", + "urllib3>=1.26.5; python_version ~= \"3.12\"", ] files = [ {file = "pinecone_client-3.2.2-py3-none-any.whl", hash = "sha256:7e492fdda23c73726bc0cb94c689bb950d06fb94e82b701a0c610c2e830db327"}, @@ -2820,8 +2845,10 @@ dependencies = [ "environs<=9.5.0", "grpcio>=1.49.1", "milvus-lite<2.5.0,>=2.4.0; sys_platform != \"win32\"", + "numpy<1.25.0; python_version <= \"3.8\"", "pandas>=1.2.4", "protobuf>=3.20.0", + "setuptools<70.1; python_version <= \"3.8\"", "setuptools>69", "ujson>=2.0.0", ] @@ -2837,6 +2864,7 @@ requires_python = ">=3.6" summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" groups = ["default"] dependencies = [ + "dataclasses; python_version < \"3.7\"", "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ @@ -2976,7 +3004,9 @@ dependencies = [ "grpcio-tools>=1.41.0", "grpcio>=1.41.0", "httpx[http2]>=0.20.0", + "numpy<1.21; python_version < \"3.8\"", "numpy>=1.21; python_version >= \"3.8\" and python_version < \"3.12\"", + "numpy>=1.26; python_version >= \"3.12\"", "portalocker<3.0.0,>=2.7.0", "pydantic>=1.10.8", "urllib3<3,>=1.26.14", @@ -2994,6 +3024,8 @@ summary = "Python client for Redis database and key-value store" groups = ["default"] dependencies = [ "async-timeout>=4.0.3; python_full_version < \"3.11.3\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, @@ -3355,6 +3387,7 @@ summary = "Database Abstraction Library" groups = ["default"] dependencies = [ "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"", + "importlib-metadata; python_version < \"3.8\"", "typing-extensions>=4.6.0", ] files = [ @@ -3535,19 +3568,6 @@ files = [ {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, @@ -3614,6 +3634,7 @@ groups = ["default"] dependencies = [ "mypy-extensions>=0.3.0", "typing-extensions>=3.7.4", + "typing>=3.7.4; python_version < \"3.5\"", ] files = [ {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, @@ -3674,11 +3695,6 @@ files = [ {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbd8fd427f57a03cff3ad6574b5e299131585d9727c8c366da4624a9069ed746"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beeaf1c48e32f07d8820c705ff8e645f8afa690cca1544adba4ebfa067efdc88"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:baed37ea46d756aca2955e99525cc02d9181de67f25515c468856c38d52b5f3b"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7663960f08cd5a2bb152f5ee3992e1af7690a64c0e26d31ba7b3ff5b2ee66337"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d8640fb4072d36b08e95a3a380ba65779d356b2fee8696afeb7794cf0902d0a1"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78778a3aa7aafb11e7ddca4e29f46bc5139131037ad628cc10936764282d6753"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0111b27f2d5c820e7f2dbad7d48e3338c824e7ac4d2a12da3dc6061cc39c8e6"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c66962ca7565605b355a9ed478292da628b8f18c0f2793021ca4425abf8b01e5"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba43cc34cce49cf2d4bc76401a754a81202d8aa926d0e2b79f0ee258cb15d3a4"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac56eb983edce27e7f51d05bc8dd820586c6e6be1c5216a6809b0c668bb312b8"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44bd4b23a0e723bf8b10628288c2c7c335161d6840013d4d5de20e48551773b"}, diff --git a/run-platform.sh b/run-platform.sh index c906bf048..510ff9c9a 100755 --- a/run-platform.sh +++ b/run-platform.sh @@ -137,22 +137,21 @@ do_git_pull() { return fi - current_version=$(git describe --tags --abbrev=0) echo "Fetching release tags." + current_version=$(git describe --tags --abbrev=0) git fetch --quiet --tags if [[ "$opt_version" == "latest" ]]; then target_branch=`git ls-remote --tags origin | awk -F/ '{print $3}' | sort -V | tail -n1` - elif [[ "$opt_version" == "main" ]]; then - target_branch="main" + elif [[ "$opt_version" == "current" ]]; then + target_branch=`git branch --show-current` opt_build_local=true - echo -e "Choosing ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""main""$default_text"" branch." + echo -e "Opting ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""$target_branch""$default_text"" branch." elif [ -z $(git tag -l "$opt_version") ]; then echo -e "$red_text""Version not found.""$default_text" - version_regex="^v([0-9]+)\.([0-9]+)\.([0-9]+)(-[a-zA-Z0-9]+(\.[0-9]+)?)?$" if [[ ! $opt_version =~ $version_regex ]]; then - echo -e "$red_text""Version must be provided with a 'v' prefix and follow SemVer (e.g. v0.47.0).""$default_text" + echo -e "$red_text""Version must be prefixed with 'v' and follow SemVer (e.g. v0.47.0).""$default_text" fi exit 1 else @@ -166,21 +165,21 @@ do_git_pull() { git pull --quiet $(git remote) $target_branch } -copy_or_merge_envs() { +_copy_or_merge_env() { + local src_file_path="$1" + local dest_file_path="$2" + local service="$3" - local src_file="$1" - local dest_file="$2" - local displayed_reason="$3" - - if [ ! -e "$dest_file" ]; then - cp "$src_file" "$dest_file" - echo -e "Created env for ""$blue_text""$displayed_reason""$default_text"" at ""$blue_text""$dest_file""$default_text""." + if [ -e "$src_file_path" ] && [ ! -e "$dest_file_path" ]; then + first_setup=true + cp "$src_file_path" "$dest_file_path" + echo -e "Created env for ""$blue_text""$service""$default_text"" at ""$blue_text""$dest_file_path""$default_text""." elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then - python3 $script_dir/docker/scripts/merge_env.py "$src_file" "$dest_file" + python3 $script_dir/docker/scripts/merge_env.py "$src_file_path" "$dest_file_path" if [ $? -ne 0 ]; then exit 1 fi - echo -e "Merged env for ""$blue_text""$displayed_reason""$default_text"" at ""$blue_text""$dest_file""$default_text""." + echo -e "Merged env for ""$blue_text""$service""$default_text"" at ""$blue_text""$dest_file_path""$default_text""." fi } @@ -193,10 +192,9 @@ setup_env() { sample_env_path="$script_dir/$service/sample.env" env_path="$script_dir/$service/.env" - if [ -e "$sample_env_path" ] && [ ! -e "$env_path" ]; then - first_setup=true - cp "$sample_env_path" "$env_path" + _copy_or_merge_env $sample_env_path $env_path $service + if [ "$first_setup" = true ]; then # Add encryption secret for backend and platform-service. if [[ "$service" == "backend" || "$service" == "platform-service" ]]; then echo -e "$blue_text""Adding encryption secret to $service""$default_text" @@ -221,19 +219,11 @@ setup_env() { # sed -i "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path fi fi - echo -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"." - elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then - python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path - if [ $? -ne 0 ]; then - exit 1 - fi - echo -e "Merged env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"." fi done - copy_or_merge_envs "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" - copy_or_merge_envs "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" - + _copy_or_merge_env "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" + _copy_or_merge_env "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" if [ "$opt_only_env" = true ]; then echo -e "$green_text""Done.""$default_text" && exit 0 @@ -244,7 +234,7 @@ build_services() { pushd ${script_dir}/docker 1>/dev/null if [ "$opt_build_local" = true ]; then - echo -e "$blue_text""Building""$default_text"" docker images ""$blue_text""$opt_version""$default_text"" locally." + echo -e "$blue_text""Building""$default_text"" docker images tag ""$blue_text""$opt_version""$default_text"" locally." VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml build || { echo -e "$red_text""Failed to build docker images.""$default_text" exit 1 diff --git a/unstract/flags/README.md b/unstract/flags/README.md index ec1da3d32..52f131fa4 100644 --- a/unstract/flags/README.md +++ b/unstract/flags/README.md @@ -1,27 +1,14 @@ # FEATURE FLAGS -## Prerequist knowlege: - To effectively use this repository, it's beneficial to have a foundational understanding of basic gRPC functionality and experience working with gRPC in Python. Familiarity with concepts such as gRPC services, protocol buffers, and the structure of gRPC-based applications will enhance your ability to leverage the tools and code provided here. +## Prerequisites - -""" -This code demonstrates the usage of feature flags with the Flipt service running as a Docker container. +To effectively use this repository, it's beneficial to have a foundational understanding of basic gRPC functionality and experience working with gRPC in Python. Familiarity with concepts such as gRPC services, protocol buffers, and the structure of gRPC-based applications will enhance your ability to leverage the tools and code provided here. Feature flags are a software development technique that allows you to enable or disable certain features in your application without deploying new code. Flipt is a feature flagging service that provides a centralized way to manage and control feature flags. -In this code, we are using Flipt to check the status of a feature flag before executing a specific block of code. The Flipt service is assumed to be running as a Docker container. - -To use this code, make sure you have the Flipt service running as a Docker container and configure the necessary connection details in the code. Then, you can use the `is_feature_enabled` function to check the status of a feature flag before executing the corresponding code. - -Note: This code assumes that you have already set up the necessary feature flags and configurations in the Flipt service. - -For more information on feature flags and Flipt, refer to the documentation: - -- Feature flags: https://en.wikipedia.org/wiki/Feature_toggle -- Flipt: https://flipt.io/ - -""" +The following code demonstrates the usage of feature flags with the Flipt service running as a Docker container. +```python def is_feature_enabled(feature_flag: str) -> bool: """ Check if a feature flag is enabled. @@ -42,31 +29,44 @@ if is_feature_enabled("my_feature"): else: # Execute code for disabled feature pass +``` + +In this code, we are using Flipt to check the status of a feature flag before executing a specific block of code. The Flipt service is assumed to be running as a Docker container. + +To use this code, make sure you have the Flipt service running as a Docker container and configure the necessary connection details in the code. Then, you can use the `is_feature_enabled` function to check the status of a feature flag before executing the corresponding code. + +Note: This code assumes that you have already set up the necessary feature flags and configurations in the Flipt service. + +For more information on feature flags and Flipt, refer to the documentation: + +- Feature flags: https://en.wikipedia.org/wiki/Feature_toggle +- Flipt: https://flipt.io/ ## Feature flags in Unstract -- Refer related files in /backend/feature_flag and /unstract/flags +- Refer related files in `/backend/feature_flag` and `/unstract/flags` -- Set required variables in backend .env to utilize feature flags: +- Set required variables in backend `.env` to utilize feature flags: + ```bash EVALUATION_SERVER_IP= EVALUATION_SERVER_PORT= + EVALUATION_SERVER_WARNINGS="true|false" + ``` - These variables has to set the FLIPT server values. - -- Generate python files for gRPC using .proto file inside feature_flags app with protoc - The generated files have suffix with pb, indicating proto buffer files +- Generate python files for gRPC using `.proto` file inside `feature_flags` app with `protoc`. + The generated files have prefix `_pb`, indicating protocol buffers code. - ``` - + ```bash python -m grpc_tools.protoc -I protos --python_out=. --grpc_python_out=. protos/evaluation.proto protoc --python_out=./protos protos/evaluation.proto - ``` -## Explanation: - The first command (python -m grpc_tools.protoc ...) generates the gRPC Python files (_pb2.py and _pb2_grpc.py) based on the evaluation.proto file located in the protos directory. - The second command (protoc --python_out=./protos ...) generates the standard Python files (_pb2.py) based on the same evaluation.proto file. +## Explanation + +The first command (`python -m grpc_tools.protoc ...`) generates the gRPC Python files (`_pb2.py` and `_pb2_grpc.py`) based on the `evaluation.proto` file located in the `protos` directory. + +The second command (`protoc --python_out=./protos ...`) generates the standard Python files (`_pb2.py`) based on the same `evaluation.proto` file. - After running the compilation commands, you can use the generated files in your Python project to implement gRPC client and server functionality. Refer to the generated files for the specific classes and methods available for use. +After running the compilation commands, you can use the generated files in your Python project to implement gRPC client and server functionality. Refer to the generated files for the specific classes and methods available for use. diff --git a/unstract/flags/src/unstract/flags/client/base.py b/unstract/flags/src/unstract/flags/client/base.py index 8ecf88fa2..548d7347d 100644 --- a/unstract/flags/src/unstract/flags/client/base.py +++ b/unstract/flags/src/unstract/flags/client/base.py @@ -7,6 +7,9 @@ class BaseClient: def __init__(self, stub_class) -> None: evaluation_server_ip = os.environ.get("EVALUATION_SERVER_IP", "") evaluation_server_port = os.environ.get("EVALUATION_SERVER_PORT", "") + evaluation_server_warnings = os.environ.get( + "EVALUATION_SERVER_WARNINGS", "false" + ) if not evaluation_server_ip: raise ValueError("No response from server, refer README.md.") @@ -15,3 +18,4 @@ def __init__(self, stub_class) -> None: f"{evaluation_server_ip}:{evaluation_server_port}" ) self.stub = stub_class(self.channel) + self.warnings = evaluation_server_warnings.lower() == "true" diff --git a/unstract/flags/src/unstract/flags/client/evaluation.py b/unstract/flags/src/unstract/flags/client/evaluation.py index 537abe869..6b9d2de17 100644 --- a/unstract/flags/src/unstract/flags/client/evaluation.py +++ b/unstract/flags/src/unstract/flags/client/evaluation.py @@ -52,14 +52,17 @@ def boolean_evaluate_feature_flag( return bool(response.enabled) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: - logger.warning( - f"Flag key {flag_key} not found in namespace {namespace_key}." - ) + if self.warnings: + logger.warning( + f"Flag key {flag_key} not found in namespace {namespace_key}." + ) elif e.code() == grpc.StatusCode.UNAVAILABLE: - logger.warning(f"Evaluation server is unavailable: {e.details()}.") + if self.warnings: + logger.warning(f"Evaluation server is unavailable: {e.details()}.") else: - logger.warning( - f"Error evaluating feature flag {flag_key} for {namespace_key}" - f" : {str(e)}" - ) + if self.warnings: + logger.warning( + f"Error evaluating feature flag {flag_key} for {namespace_key}" + f" : {str(e)}" + ) return False