diff --git a/README.md b/README.md index 565e84400..bf0d32ff9 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,14 @@ We just need to override default Traefik proxy routing to allow this, that's all Run the services. +#### Generate Encryption key to be used in backend and Platform service + + Generate Fernet Key Refer https://pypi.org/project/cryptography/ + + `ENCRYPTION_KEY=$(python -c "import secrets, base64; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).decode())")` + + use the above generated encryption, key in ENV's of platform and backend + #### Conflicting Host Names When same host name environment variables are used by both the service running locally and a service diff --git a/backend/account/migrations/0005_encryptionsecret.py b/backend/account/migrations/0005_encryptionsecret.py index d8ff25b8a..724c12522 100644 --- a/backend/account/migrations/0005_encryptionsecret.py +++ b/backend/account/migrations/0005_encryptionsecret.py @@ -1,9 +1,5 @@ # Generated by Django 4.2.1 on 2024-02-13 11:52 -from typing import Any - -from account.models import EncryptionSecret -from cryptography.fernet import Fernet from django.db import migrations, models @@ -12,11 +8,6 @@ class Migration(migrations.Migration): ("account", "0004_alter_platformkey_key_name_and_more"), ] - def initialize_secret(apps: Any, schema_editor: Any) -> None: - EncryptionSecret.objects.create( - key=Fernet.generate_key().decode("utf-8") - ) - operations = [ migrations.CreateModel( name="EncryptionSecret", @@ -33,7 +24,4 @@ def initialize_secret(apps: Any, schema_editor: Any) -> None: ("key", models.CharField(blank=True, max_length=64)), ], ), - migrations.RunPython( - initialize_secret, reverse_code=migrations.RunPython.noop - ), ] diff --git a/backend/account/migrations/0006_delete_encryptionsecret.py b/backend/account/migrations/0006_delete_encryptionsecret.py new file mode 100644 index 000000000..1216373ee --- /dev/null +++ b/backend/account/migrations/0006_delete_encryptionsecret.py @@ -0,0 +1,15 @@ +# Generated by Django 4.2.1 on 2024-03-04 05:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("account", "0005_encryptionsecret"), + ] + + operations = [ + migrations.DeleteModel( + name="EncryptionSecret", + ), + ] diff --git a/backend/account/models.py b/backend/account/models.py index c3bafc8fb..63dade30d 100644 --- a/backend/account/models.py +++ b/backend/account/models.py @@ -1,10 +1,11 @@ import uuid -from backend.constants import FieldLengthConstants as FieldLength from django.contrib.auth.models import AbstractUser, Group, Permission from django.db import models from django_tenants.models import DomainMixin, TenantMixin +from backend.constants import FieldLengthConstants as FieldLength + NAME_SIZE = 64 KEY_SIZE = 64 @@ -131,11 +132,3 @@ class Meta: name="unique_key_name", ), ] - - -class EncryptionSecret(models.Model): - key = models.CharField( - max_length=KEY_SIZE, - null=False, - blank=True, - ) diff --git a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py index 5f65c0040..d5a9bb4cf 100644 --- a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py +++ b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py @@ -3,21 +3,20 @@ import json from typing import Any -from account.models import EncryptionSecret from adapter_processor.models import AdapterInstance from cryptography.fernet import Fernet +from django.conf import settings from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("adapter_processor", "0002_adapterinstance_unique_adapter"), - ("account", "0005_encryptionsecret"), ] def EncryptCredentials(apps: Any, schema_editor: Any) -> None: - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) queryset = AdapterInstance.objects.all() for obj in queryset: # type: ignore diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.py index bb2443450..577cd5ffb 100644 --- a/backend/adapter_processor/serializers.py +++ b/backend/adapter_processor/serializers.py @@ -1,10 +1,10 @@ import json from typing import Any -from account.models import EncryptionSecret from adapter_processor.adapter_processor import AdapterProcessor from adapter_processor.constants import AdapterKeys from cryptography.fernet import Fernet +from django.conf import settings from rest_framework import serializers from unstract.adapters.constants import Common as common @@ -45,8 +45,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer): """ def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]: - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA)) data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt( @@ -58,8 +58,8 @@ def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]: def to_representation(self, instance: AdapterInstance) -> dict[str, str]: rep: dict[str, str] = super().to_representation(instance) - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) rep.pop(AdapterKeys.ADAPTER_METADATA_B) adapter_metadata = json.loads( diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 4252b1483..9b9deea2f 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -143,6 +143,7 @@ def get_required_setting( # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = get_required_setting("DJANGO_SECRET_KEY") +ENCRYPTION_KEY = get_required_setting("ENCRYPTION_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py index 7f9090a2e..1e4a342c9 100644 --- a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py +++ b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py @@ -3,21 +3,20 @@ import json from typing import Any -from account.models import EncryptionSecret from connector.models import ConnectorInstance from cryptography.fernet import Fernet +from django.conf import settings from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("connector", "0001_initial"), - ("account", "0005_encryptionsecret"), ] def EncryptCredentials(apps: Any, schema_editor: Any) -> None: - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) queryset = ConnectorInstance.objects.all() for obj in queryset: # type: ignore diff --git a/backend/connector/models.py b/backend/connector/models.py index bc37af041..eb0a3f7da 100644 --- a/backend/connector/models.py +++ b/backend/connector/models.py @@ -2,12 +2,13 @@ import uuid from typing import Any -from account.models import EncryptionSecret, User +from account.models import User from connector.fields import ConnectorAuthJSONField from connector_auth.models import ConnectorAuth from connector_processor.connector_processor import ConnectorProcessor from connector_processor.constants import ConnectorKeys from cryptography.fernet import Fernet +from django.conf import settings from django.db import models from project.models import Project from utils.models.base_model import BaseModel @@ -112,8 +113,8 @@ def __str__(self) -> str: @property def metadata(self) -> Any: - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - cipher_suite: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8")) decrypted_value = cipher_suite.decrypt( bytes(self.connector_metadata_b).decode("utf-8") ) diff --git a/backend/connector/serializers.py b/backend/connector/serializers.py index 4b664cb67..820b47d66 100644 --- a/backend/connector/serializers.py +++ b/backend/connector/serializers.py @@ -3,7 +3,6 @@ from collections import OrderedDict from typing import Any, Optional -from account.models import EncryptionSecret from connector.constants import ConnectorInstanceKey as CIKey from connector_auth.models import ConnectorAuth from connector_auth.pipeline.common import ConnectorAuthHelper @@ -11,6 +10,7 @@ from connector_processor.constants import ConnectorKeys from connector_processor.exceptions import OAuthTimeOut from cryptography.fernet import Fernet +from django.conf import settings from utils.serializer_utils import SerializerUtils from backend.serializers import AuditSerializer @@ -55,8 +55,8 @@ def save(self, **kwargs): # type: ignore ) kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) json_string: str = json.dumps(kwargs.pop(CIKey.CONNECTOR_METADATA)) if self.validated_data: self.validated_data.pop(CIKey.CONNECTOR_METADATA) @@ -81,8 +81,8 @@ def to_representation(self, instance: ConnectorInstance) -> dict[str, str]: ] = ConnectorProcessor.get_connector_data_with_key( instance.connector_id, ConnectorKeys.ICON ) - encryption_secret: EncryptionSecret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) rep.pop(CIKey.CONNECTOR_METADATA_B) if instance.connector_metadata_b: diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py index 25f4b215b..150d97371 100644 --- a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py +++ b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py @@ -4,15 +4,16 @@ import django.db.models.deletion from cryptography.fernet import Fernet +from django.conf import settings from django.db import connection, migrations, models def fill_with_default_x2text(apps, schema): ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager") AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance") - EncryptionSecret = apps.get_model("account", "EncryptionSecret") - encryption_secret = EncryptionSecret.objects.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + + encryption_secret: str = settings.ENCRYPTION_KEY + f: Fernet = Fernet(encryption_secret.encode("utf-8")) metadata = { "url": "http://unstract-unstructured-io:8000/general/v0/general" } @@ -44,7 +45,6 @@ def disable_triggers(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ("account", "0005_encryptionsecret"), ("adapter_processor", "0004_alter_adapterinstance_adapter_type"), ( "prompt_profile_manager", diff --git a/backend/sample.env b/backend/sample.env index 1ffd6a168..5fca2ee4d 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -100,3 +100,7 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python #X2Text Service X2TEXT_HOST=http://unstract-x2text-service X2TEXT_PORT=3004 + +# Encryption Key +# key must be 32 url-safe base64-encoded bytes. +ENCRYPTION_KEY="Sample-Key" diff --git a/platform-service/sample.env b/platform-service/sample.env index 03b2a1051..3d8a69a13 100644 --- a/platform-service/sample.env +++ b/platform-service/sample.env @@ -23,3 +23,7 @@ PG_V_PORT=5432 PG_V_USERNAME=unstract_dev PG_V_PASSWORD=unstract_pass PG_V_DATABASE=unstract_db + +# Encryption Key +# key must be 32 url-safe base64-encoded bytes. +ENCRYPTION_KEY="Sample-Key" diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.py index 8aad19928..627343055 100644 --- a/platform-service/src/unstract/platform_service/main.py +++ b/platform-service/src/unstract/platform_service/main.py @@ -45,6 +45,7 @@ PG_V_USERNAME = os.environ.get("PG_V_USERNAME", "user") PG_V_PASSWORD = os.environ.get("PG_V_PASSWORD", "") PG_V_DATABASE = os.environ.get("PG_V_DATABASE", "") +ENCRYPTION_KEY = os.environ.get("ENCRYPTION_KEY") if not (REDIS_HOST and REDIS_PORT): raise ValueError( "REDIS_HOST and REDIS_PORT must be set in the environment." @@ -74,19 +75,6 @@ be_db.connect() -class EncryptionSecret(peewee.Model): - id = peewee.IntegerField() - key = peewee.CharField() - - class Meta: - table_name = "account_encryptionsecret" - database = be_db # This model uses the "BE_DB" database. - - def save(self, force_insert: bool = False, only: Any = None) -> Any: - self.modified_at = datetime.datetime.now() - return super().save(force_insert=force_insert, only=only) - - class UnstractUsage(peewee.Model): id = peewee.UUIDField(primary_key=True, default=uuid.uuid4) created_at = peewee.DateTimeField(default=datetime.datetime.now) @@ -408,8 +396,7 @@ def adapter_instance() -> Any: ) ) - encryption_secret: EncryptionSecret = EncryptionSecret.get() - f: Fernet = Fernet(encryption_secret.key.encode("utf-8")) + f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8")) data_dict["adapter_metadata"] = json.loads( f.decrypt(