Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

some fixes (Sourcery refactored) #7

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion imoog/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
# dummy __init__
# dummy __init__
42 changes: 28 additions & 14 deletions imoog/app.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
from __future__ import annotations

import importlib
from typing import Tuple
from typing import Tuple, Type

from starlette.applications import Starlette
from starlette.routing import Route, Mount
from starlette.routing import Route
from starlette.middleware.cors import CORSMiddleware
from starlette.middleware.trustedhost import TrustedHostMiddleware
from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware
Expand All @@ -15,7 +15,7 @@
delete_file
)
from imoog import settings

from imoog.database.drivers import Driver

routes = [
Route(
Expand All @@ -36,7 +36,7 @@
]

app = Starlette(routes=routes)
app.add_middleware(CORSMiddleware, allow_origins = settings.CORS_ALLOWED_ORIGINS)
app.add_middleware(CORSMiddleware, allow_origins=settings.CORS_ALLOWED_ORIGINS)
app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS)

if settings.ENFORCE_SECURE_SCHEME is True:
Expand All @@ -45,33 +45,45 @@
app.image_cache = None
app.db_driver = None


def _check_driver() -> Tuple[type, str]:
_driver_path = settings.DATABASE_DRIVERS["driver"]
package = importlib.import_module(_driver_path)
driver = package._DRIVER
_type = package._DRIVER_TYPE
return (driver, _type)

# noinspection PyUnresolvedReferences
driver: Type[Driver] = package.DB_DRIVER
# noinspection PyUnresolvedReferences
driver_type: str = package.DB_DRIVER_TYPE

return driver, driver_type


def _check_cache_driver() -> Tuple[type, str]:
_driver_path = settings.CACHE_DRIVERS["driver"]
_driver_path: str = settings.CACHE_DRIVERS.get("driver")
package = importlib.import_module(_driver_path)
driver = package._DRIVER
_type = package._DRIVER_TYPE
return (driver, _type)


# noinspection PyUnresolvedReferences
driver = package.C_DRIVER
# noinspection PyUnresolvedReferences
_type = package.C_DRIVER_TYPE

return driver, _type


@app.on_event("startup")
async def on_startup():
# connect to databases and ready caches.
driver_class, _ = _check_driver()
config = settings.DATABASE_DRIVERS["config"]

driver = driver_class()
await driver.connect(**config)

cache_driver_class, _ = _check_cache_driver()
cache_config = settings.CACHE_DRIVERS["config"]
cache_driver = cache_driver_class()

cache_config["max_cache_size"] = settings.MAX_CACHE_SIZE # we pass this into
cache_config["max_cache_size"] = settings.MAX_CACHE_SIZE # we pass this into
# the connect function of the cache driver regardless whether its the
# memory cache driver or not.
await cache_driver.connect(**cache_config)
Expand All @@ -82,9 +94,11 @@ async def on_startup():

@app.on_event("shutdown")
async def on_shutdown():
# noinspection PyBroadException
try:
await app.db_driver.cleanup()
await app.cache_driver.cleanup()
await app.image_cache.cleanup()

except Exception:
# disregard any errors that occur
# within the driver cleanup
Expand Down
3 changes: 2 additions & 1 deletion imoog/cache/basecache.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
class Cache:
# a base cache handler

def __init__(self):
self._connection = None # this is similar
self._connection = None # this is similar
# to the database handlers, except this can also just
# be a regular dictionary

Expand Down
47 changes: 26 additions & 21 deletions imoog/cache/memorycache.py
Original file line number Diff line number Diff line change
@@ -1,53 +1,58 @@
from typing import Dict
from typing import Dict, Union

from .basecache import Cache


class InMemoryCache(Cache):
_max_size: int

async def connect(self, **kwargs):
# to keep it consistent
# we will indeed have an async function
# to just register a dict
self._connection: Dict[str, bytes] = {}
max_size = kwargs["max_cache_size"]
self._connection: Dict[str, Union[bytes, str]] = {}
max_size = kwargs.get("max_cache_size")

self._max_size = max_size
return self._connection

async def get(self, key: str):
mime = self._connection.get(key + "__mime__")
mime = self._connection.get(f"{key}__mime__")
image = self._connection.get(key)
R = (image, mime)
if None in R:
r = (image, mime)

if None in r:
return None
return R

return r

async def delete(self, key: str) -> bool:
try:
self._connection.pop(key)
self._connection.pop(key + "__mime__")
# if the first pop fails, the second one will
# fail too as its dependent on the first one to exist
self._connection.pop(f"{key}__mime__")
# if the first pop fails, the second one will
# fail too as its dependent on the first one to exist

except KeyError:
return False # failed
return False # failed

return True # success
return True # success

async def set(
self,
key: str,
image: bytes,
mime: str
self,
key: str,
image: bytes,
mime: str
):
self._connection[key] = image
self._connection[key + "__mime__"] = mime
self._connection[f"{key}__mime__"] = mime

async def cleanup(self):
# believe it or not
# there is a cleanup operation for a dictionary
# we'll free the memory by clearing the dictionary
self._connection.clear()

_DRIVER = InMemoryCache
_DRIVER_TYPE = "MEMORY"

C_DRIVER = InMemoryCache
C_DRIVER_TYPE = "MEMORY"
25 changes: 16 additions & 9 deletions imoog/cache/rediscache.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,30 @@ async def connect(self, **kwargs):
async def get(self, image: str):
futures = [
self._connection.get(image),
self._connection.get(image + "__mime__")
self._connection.get(f"{image}__mime__"),
]
R = await asyncio.gather(*futures)

if None in R:
r = await asyncio.gather(*futures)

if None in r:
return None
return tuple(R)

return tuple(r)

async def delete(self, image: str) -> bool:
futures = [
self._connection.delete(image),
self._connection.delete(image + "__mime__")
self._connection.delete(f"{image}__mime__"),
]


# noinspection PyBroadException
try:
await asyncio.gather(*futures)

except Exception:
return False

else:
return True

Expand All @@ -53,12 +58,14 @@ async def set(
):
futures = [
self._connection.set(key, image),
self._connection.set(key + "__mime__", mime)
self._connection.set(f"{key}__mime__", mime),
]

await asyncio.gather(*futures)

async def cleanup(self):
return await self._connection.close()

_DRIVER = RedisCache
_DRIVER_TYPE = "REDIS"

C_DRIVER = RedisCache
C_DRIVER_TYPE = "REDIS"
7 changes: 2 additions & 5 deletions imoog/database/drivers.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import zlib
from typing import Any


class Driver:
def __init__(self):
self._connection = None # this connection instance
self.pool = None # this connection instance
# is filled in the connect method.
self.identifier = None # this is custom per database driver.
# this attribute will be None until the connect method is called.
Expand All @@ -23,5 +21,4 @@ def set_custom_val(self, key: str, value: str):
return {key: value}

def decompress(self, _bytes: bytes) -> bytes:
decompressed = zlib.decompress(_bytes)
return decompressed
return zlib.decompress(_bytes)
26 changes: 18 additions & 8 deletions imoog/database/mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,14 @@
AsyncIOMotorClient
)

from .drivers import Driver # import the base driver impl
from .drivers import Driver # import the base driver impl


class MongoDriver(Driver):
TABLE_NAME: str
_connection: AsyncIOMotorCollection
_parent_client: AsyncIOMotorClient

async def connect(self, **kwargs):
self.identifier = "mongo"

Expand All @@ -37,45 +41,51 @@ async def insert(
image: bytes,
name: str,
mime: str
):
) -> int:
insert = {
"_id": name,
"image": image,
"mime": mime
}
await self._connection.insert_one(insert)
return 0
return 0 # again, why?

async def fetch(
self,
name: str
) -> Tuple[bytes, str]:
query = {"_id": name}
result = await self._connection.find_one(query)

image = result["image"]
mime = result["mime"]
decompressed = self.decompress(image)
return (decompressed, mime)

return decompressed, mime

async def delete(
self,
name: str
) -> bool:
# Returns whether the delete succeeded or failed
query = {"_id": name}
# noinspection PyBroadException
try:
await self._connection.delete_one(query)

except Exception:
return False

else:
return True

async def fetch_all(self) -> Tuple[List[Mapping[str, Any]], str]:
documents = await self._connection.find({}).to_list(length=99999999999999999999) # big number
return (documents, "_id")
documents = await self._connection.find({}).to_list(length=99999999999999999999) # big number
return documents, "_id"

async def cleanup(self):
return self._parent_client.close()

_DRIVER = MongoDriver
_DRIVER_TYPE = "MONGO"

DB_DRIVER = MongoDriver
DB_DRIVER_TYPE = "MONGO"
Loading