Moved away from storing encryption key in database to ENV (#39)

* Moved away from storing it in database

* Platform service changes

* Removed encryption model and its added dependencies

* Updated the sample env for the encryption part

* Updated the sample env for the encryption part

* Updated the sample env for the encryption part

* Updated the read me with scripts

---------

Co-authored-by: Neha <[email protected]>

README.md

@@ -176,6 +176,14 @@ We just need to override default Traefik proxy routing to allow this, that's all
 
 Run the services.
 
+#### Generate Encryption key to be used in backend and Platform service
+
+ Generate Fernet Key Refer https://pypi.org/project/cryptography/
+
+ `ENCRYPTION_KEY=$(python -c "import secrets, base64; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).decode())")`
+
+ use the above generated encryption, key in ENV's of platform and backend
+
 #### Conflicting Host Names
 
 When same host name environment variables are used by both the service running locally and a service

backend/account/migrations/0005_encryptionsecret.py

@@ -1,9 +1,5 @@
 # Generated by Django 4.2.1 on 2024-02-13 11:52
 
-from typing import Any
-
-from account.models import EncryptionSecret
-from cryptography.fernet import Fernet
 from django.db import migrations, models
 
 
@@ -12,11 +8,6 @@ class Migration(migrations.Migration):
         ("account", "0004_alter_platformkey_key_name_and_more"),
     ]
 
-    def initialize_secret(apps: Any, schema_editor: Any) -> None:
-        EncryptionSecret.objects.create(
-            key=Fernet.generate_key().decode("utf-8")
-        )
-
     operations = [
         migrations.CreateModel(
             name="EncryptionSecret",
@@ -33,7 +24,4 @@ def initialize_secret(apps: Any, schema_editor: Any) -> None:
                 ("key", models.CharField(blank=True, max_length=64)),
             ],
         ),
-        migrations.RunPython(
-            initialize_secret, reverse_code=migrations.RunPython.noop
-        ),
     ]

backend/account/migrations/0006_delete_encryptionsecret.py

@@ -0,0 +1,15 @@
+# Generated by Django 4.2.1 on 2024-03-04 05:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("account", "0005_encryptionsecret"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="EncryptionSecret",
+        ),
+    ]

backend/account/models.py

@@ -1,10 +1,11 @@
 import uuid
 
-from backend.constants import FieldLengthConstants as FieldLength
 from django.contrib.auth.models import AbstractUser, Group, Permission
 from django.db import models
 from django_tenants.models import DomainMixin, TenantMixin
 
+from backend.constants import FieldLengthConstants as FieldLength
+
 NAME_SIZE = 64
 KEY_SIZE = 64
 
@@ -131,11 +132,3 @@ class Meta:
                 name="unique_key_name",
             ),
         ]
-
-
-class EncryptionSecret(models.Model):
-    key = models.CharField(
-        max_length=KEY_SIZE,
-        null=False,
-        blank=True,
-    )

backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py

@@ -3,21 +3,20 @@
 import json
 from typing import Any
 
-from account.models import EncryptionSecret
 from adapter_processor.models import AdapterInstance
 from cryptography.fernet import Fernet
+from django.conf import settings
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
         ("adapter_processor", "0002_adapterinstance_unique_adapter"),
-        ("account", "0005_encryptionsecret"),
     ]
 
     def EncryptCredentials(apps: Any, schema_editor: Any) -> None:
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
         queryset = AdapterInstance.objects.all()
 
         for obj in queryset:  # type: ignore

backend/adapter_processor/serializers.py

@@ -1,10 +1,10 @@
 import json
 from typing import Any
 
-from account.models import EncryptionSecret
 from adapter_processor.adapter_processor import AdapterProcessor
 from adapter_processor.constants import AdapterKeys
 from cryptography.fernet import Fernet
+from django.conf import settings
 from rest_framework import serializers
 from unstract.adapters.constants import Common as common
 
@@ -45,8 +45,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):
     """
 
     def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
         json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA))
 
         data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt(
@@ -58,8 +58,8 @@ def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:
     def to_representation(self, instance: AdapterInstance) -> dict[str, str]:
         rep: dict[str, str] = super().to_representation(instance)
 
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
 
         rep.pop(AdapterKeys.ADAPTER_METADATA_B)
         adapter_metadata = json.loads(

backend/backend/settings/base.py

@@ -143,6 +143,7 @@ def get_required_setting(
 
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = get_required_setting("DJANGO_SECRET_KEY")
+ENCRYPTION_KEY = get_required_setting("ENCRYPTION_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True

backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py

@@ -3,21 +3,20 @@
 import json
 from typing import Any
 
-from account.models import EncryptionSecret
 from connector.models import ConnectorInstance
 from cryptography.fernet import Fernet
+from django.conf import settings
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
         ("connector", "0001_initial"),
-        ("account", "0005_encryptionsecret"),
     ]
 
     def EncryptCredentials(apps: Any, schema_editor: Any) -> None:
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
         queryset = ConnectorInstance.objects.all()
 
         for obj in queryset:  # type: ignore

backend/connector/models.py

@@ -2,12 +2,13 @@
 import uuid
 from typing import Any
 
-from account.models import EncryptionSecret, User
+from account.models import User
 from connector.fields import ConnectorAuthJSONField
 from connector_auth.models import ConnectorAuth
 from connector_processor.connector_processor import ConnectorProcessor
 from connector_processor.constants import ConnectorKeys
 from cryptography.fernet import Fernet
+from django.conf import settings
 from django.db import models
 from project.models import Project
 from utils.models.base_model import BaseModel
@@ -112,8 +113,8 @@ def __str__(self) -> str:
 
     @property
     def metadata(self) -> Any:
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        cipher_suite: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))
         decrypted_value = cipher_suite.decrypt(
             bytes(self.connector_metadata_b).decode("utf-8")
         )

backend/connector/serializers.py

@@ -3,14 +3,14 @@
 from collections import OrderedDict
 from typing import Any, Optional
 
-from account.models import EncryptionSecret
 from connector.constants import ConnectorInstanceKey as CIKey
 from connector_auth.models import ConnectorAuth
 from connector_auth.pipeline.common import ConnectorAuthHelper
 from connector_processor.connector_processor import ConnectorProcessor
 from connector_processor.constants import ConnectorKeys
 from connector_processor.exceptions import OAuthTimeOut
 from cryptography.fernet import Fernet
+from django.conf import settings
 from utils.serializer_utils import SerializerUtils
 
 from backend.serializers import AuditSerializer
@@ -55,8 +55,8 @@ def save(self, **kwargs):  # type: ignore
         )
         kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value
 
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
         json_string: str = json.dumps(kwargs.pop(CIKey.CONNECTOR_METADATA))
         if self.validated_data:
             self.validated_data.pop(CIKey.CONNECTOR_METADATA)
@@ -81,8 +81,8 @@ def to_representation(self, instance: ConnectorInstance) -> dict[str, str]:
             ] = ConnectorProcessor.get_connector_data_with_key(
                 instance.connector_id, ConnectorKeys.ICON
             )
-        encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
-        f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+        encryption_secret: str = settings.ENCRYPTION_KEY
+        f: Fernet = Fernet(encryption_secret.encode("utf-8"))
 
         rep.pop(CIKey.CONNECTOR_METADATA_B)
         if instance.connector_metadata_b:

backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py

@@ -4,15 +4,16 @@
 
 import django.db.models.deletion
 from cryptography.fernet import Fernet
+from django.conf import settings
 from django.db import connection, migrations, models
 
 
 def fill_with_default_x2text(apps, schema):
     ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager")
     AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")
-    EncryptionSecret = apps.get_model("account", "EncryptionSecret")
-    encryption_secret = EncryptionSecret.objects.get()
-    f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+
+    encryption_secret: str = settings.ENCRYPTION_KEY
+    f: Fernet = Fernet(encryption_secret.encode("utf-8"))
     metadata = {
         "url": "http://unstract-unstructured-io:8000/general/v0/general"
     }
@@ -44,7 +45,6 @@ def disable_triggers(apps, schema_editor):
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("account", "0005_encryptionsecret"),
         ("adapter_processor", "0004_alter_adapterinstance_adapter_type"),
         (
             "prompt_profile_manager",

backend/sample.env

@@ -100,3 +100,7 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
 #X2Text Service
 X2TEXT_HOST=http://unstract-x2text-service
 X2TEXT_PORT=3004
+
+# Encryption Key
+# key must be 32 url-safe base64-encoded bytes.
+ENCRYPTION_KEY="Sample-Key"

platform-service/sample.env

@@ -23,3 +23,7 @@ PG_V_PORT=5432
 PG_V_USERNAME=unstract_dev
 PG_V_PASSWORD=unstract_pass
 PG_V_DATABASE=unstract_db
+
+# Encryption Key
+# key must be 32 url-safe base64-encoded bytes.
+ENCRYPTION_KEY="Sample-Key"

platform-service/src/unstract/platform_service/main.py

@@ -45,6 +45,7 @@
 PG_V_USERNAME = os.environ.get("PG_V_USERNAME", "user")
 PG_V_PASSWORD = os.environ.get("PG_V_PASSWORD", "")
 PG_V_DATABASE = os.environ.get("PG_V_DATABASE", "")
+ENCRYPTION_KEY = os.environ.get("ENCRYPTION_KEY")
 if not (REDIS_HOST and REDIS_PORT):
     raise ValueError(
         "REDIS_HOST and REDIS_PORT must be set in the environment."
@@ -74,19 +75,6 @@
 be_db.connect()
 
 
-class EncryptionSecret(peewee.Model):
-    id = peewee.IntegerField()
-    key = peewee.CharField()
-
-    class Meta:
-        table_name = "account_encryptionsecret"
-        database = be_db  # This model uses the "BE_DB" database.
-
-    def save(self, force_insert: bool = False, only: Any = None) -> Any:
-        self.modified_at = datetime.datetime.now()
-        return super().save(force_insert=force_insert, only=only)
-
-
 class UnstractUsage(peewee.Model):
     id = peewee.UUIDField(primary_key=True, default=uuid.uuid4)
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
@@ -408,8 +396,7 @@ def adapter_instance() -> Any:
                 )
             )
 
-            encryption_secret: EncryptionSecret = EncryptionSecret.get()
-            f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+            f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8"))
 
             data_dict["adapter_metadata"] = json.loads(
                 f.decrypt(