Add cache adapters

README.md

@@ -0,0 +1,49 @@
+# Shared Memory Dict
+A very simple [shared memory](https://docs.python.org/3/library/multiprocessing.shared_memory.html) dict implementation.
+
+**Require**: Python >=3.8
+
+```python
+>> from shared_memory_dict import SharedMemoryDict
+>> smd = SharedMemoryDict(name='tokens', size=1024)
+>> smd['some-key'] = 'some-value-with-any-type'
+>> smd['some-key']
+'some-value-with-any-type'
+```
+
+> The arg `name` defines the location of the memory block, so if you want to share the memory between process use the same name
+
+To use [uwsgidecorators.lock](https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html#uwsgidecorators.lock) on write operations of shared memory dict set environment variable `SHARED_MEMORY_USE_UWSGI_LOCK`.
+
+## Django Cache Implementation
+There's a [Django Cache Implementation](https://docs.djangoproject.com/en/3.0/topics/cache/) with Shared Memory Dict:
+
+```python
+# settings/base.py
+CACHES = {
+    'default': {
+        'BACKEND': 'shared_memory_dict.caches.django.SharedMemoryCache',
+        'LOCATION': 'memory',
+        'OPTIONS': {'MEMORY_BLOCK_SIZE': 1024}
+    }
+}
+```
+
+> This implementation is very based on django [LocMemCache](https://docs.djangoproject.com/en/3.0/topics/cache/#local-memory-caching)
+
+
+## AioCache Backend
+There's also a [AioCache Backend Implementation](https://aiocache.readthedocs.io/en/latest/caches.html) with Shared Memory Dict:
+
+```python
+From aiocache import caches
+
+caches.set_config({
+    'default': {
+        'cache': 'shared_memory_dict.caches.aiocache.SharedMemoryCache',
+        'size': 1024,
+    },
+})
+```
+
+> This implementation is very based on aiocache [SimpleMemoryCache](https://aiocache.readthedocs.io/en/latest/caches.html#simplememorycache)

shared_memory_dict/caches/aiocache.py

@@ -0,0 +1,154 @@
+import asyncio
+from asyncio.events import AbstractEventLoop, TimerHandle
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from aiocache.base import BaseCache
+from aiocache.serializers import BaseSerializer, NullSerializer
+
+from ..dict import SharedMemoryDict
+
+Number = Union[int, float]
+
+
+class SharedMemoryCache(BaseCache):
+    """
+    A AioCache implementation of SharedMemoryDict
+    based on aiocache.backends.memory.SimpleMemoryCache
+    """
+
+    NAME = 'shared_memory'
+
+    def __init__(
+        self,
+        serializer: Optional[BaseSerializer] = None,
+        name: str = 'smc',
+        size: int = 1024,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.serializer = serializer or NullSerializer()
+        self._cache = SharedMemoryDict(f'c_{name}', size)
+        self._handlers: Dict[str, TimerHandle] = {}
+
+    @classmethod
+    def parse_uri_path(cls, path: str) -> Dict:
+        return {}
+
+    async def _get(
+        self, key: str, encoding: Optional[str] = 'utf-8', _conn=None
+    ):
+        return self._cache.get(key)
+
+    async def _multi_get(
+        self, keys: List[str], encoding: Optional[str] = 'utf-8', _conn=None
+    ):
+        return [self._cache.get(key) for key in keys]
+
+    async def _set(
+        self,
+        key: str,
+        value: Any,
+        ttl: Optional[Number] = None,
+        _cas_token: Optional[Any] = None,
+        _conn=None,
+    ) -> bool:
+        if _cas_token is not None and _cas_token != self._cache.get(key):
+            return False
+
+        if key in self._handlers:
+            self._handlers[key].cancel()
+
+        self._cache[key] = value
+        if ttl:
+            self._handlers[key] = self._loop().call_later(
+                ttl, self._delete_key, key
+            )
+
+        return True
+
+    async def _multi_set(
+        self,
+        pairs: Tuple[Tuple[str, Any]],
+        ttl: Optional[Number] = None,
+        _conn=None,
+    ) -> bool:
+        for key, value in pairs:
+            await self._set(key, value, ttl=ttl)
+        return True
+
+    async def _add(
+        self, key: str, value: Any, ttl: Optional[Number] = None, _conn=None,
+    ):
+        if key in self._cache:
+            raise ValueError(
+                f'Key {key} already exists, use .set to update the value'
+            )
+
+        await self._set(key, value, ttl=ttl)
+        return True
+
+    async def _exists(self, key: str, _conn=None):
+        return key in self._cache
+
+    async def _increment(self, key: str, delta: int, _conn=None):
+        new_value = delta
+        if key not in self._cache:
+            self._cache[key] = delta
+        else:
+            value = self._cache[key]
+            try:
+                new_value = int(value) + delta
+            except ValueError:
+                raise TypeError('Value is not an integer') from None
+
+        self._cache[key] = new_value
+        return new_value
+
+    async def _expire(
+        self, key: str, ttl: Union[int, float], _conn=None
+    ) -> bool:
+        if key not in self._cache:
+            return False
+
+        handle = self._handlers.pop(key, None)
+
+        if handle:
+            handle.cancel()
+        if ttl:
+            self._handlers[key] = self._loop().call_later(
+                ttl, self._delete_key, key
+            )
+
+        return True
+
+    async def _delete(self, key: str, _conn=None) -> int:
+        return self._delete_key(key)
+
+    async def _clear(
+        self, namespace: Optional[str] = None, _conn=None
+    ) -> bool:
+        if namespace:
+            for key in self._cache.keys():
+                if key.startswith(namespace):
+                    self._delete_key(key)
+        else:
+            self._cache.clear()
+            self._handlers = {}
+        return True
+
+    async def _redlock_release(self, key: str, value: Any) -> int:
+        if self._cache.get(key) == value:
+            self._cache.pop(key)
+            return 1
+        return 0
+
+    def _delete_key(self, key: str) -> int:
+        if self._cache.pop(key, None):
+            handle = self._handlers.pop(key, None)
+            if handle:
+                handle.cancel()
+            return 1
+        return 0
+
+    def _loop(self) -> AbstractEventLoop:
+        return asyncio.get_event_loop()

shared_memory_dict/caches/django.py

@@ -0,0 +1,123 @@
+from time import time
+from typing import Any, Dict, Optional
+
+from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
+
+from ..dict import SharedMemoryDict
+
+_caches: Dict[str, SharedMemoryDict] = {}
+
+
+class SharedMemoryCache(BaseCache):
+    """
+    A Django Cache implementation of SharedMemoryDict
+
+    Values are stored as tuple with (`value`, `expiration time`)
+    """
+
+    def __init__(self, name: str, params: Dict) -> None:
+        super().__init__(params=params)
+        options = params.get('OPTIONS', {})
+        self._cache = _caches.get(
+            name,
+            SharedMemoryDict(
+                f'c_{name}', options.get('MEMORY_BLOCK_SIZE', 1024)
+            ),
+        )
+
+    def add(
+        self,
+        key: str,
+        value: Any,
+        timeout: Optional[int] = DEFAULT_TIMEOUT,
+        version: Optional[int] = None,
+    ):
+        key = self.make_key(key, version=version)
+        self.validate_key(key)
+
+        if self._has_expired(key):
+            self._set(key, value, timeout)
+            return True
+
+        return False
+
+    def get(
+        self,
+        key: str,
+        default: Optional[Any] = None,
+        version: Optional[int] = None,
+    ):
+        key = self.make_key(key, version=version)
+        self.validate_key(key)
+
+        if self._has_expired(key):
+            self._delete(key)
+            return default
+
+        value, _ = self._cache[key]
+        self._cache.move_to_end(key, last=False)
+
+        return value
+
+    def set(
+        self,
+        key: str,
+        value: Any,
+        timeout: Optional[int] = DEFAULT_TIMEOUT,
+        version: Optional[int] = None,
+    ):
+        key = self.make_key(key, version=version)
+        self.validate_key(key)
+        self._set(key, value, timeout)
+
+    def incr(
+        self, key: str, delta: Optional[int] = 1, version: Optional[int] = None
+    ):
+        key = self.make_key(key, version=version)
+        self.validate_key(key)
+
+        if self._has_expired(key):
+            self._delete(key)
+            raise ValueError(f'Key "{key}" not found')
+
+        value, expire_info = self._cache[key]
+        new_value = value + delta
+
+        self._cache[key] = (new_value, expire_info)
+        self._cache.move_to_end(key, last=False)
+
+        return new_value
+
+    def delete(self, key: str, version: Optional[int] = None) -> None:
+        key = self.make_key(key, version=version)
+        self.validate_key(key)
+        return self._delete(key)
+
+    def clear(self):
+        self._cache.clear()
+
+    def _has_expired(self, key: str) -> bool:
+        exp = self._cache.get(key, (None, -1))[1]
+        return exp is not None and exp <= time()
+
+    def _set(
+        self, key: str, value: Any, timeout: Optional[int] = DEFAULT_TIMEOUT
+    ):
+        if len(self._cache) >= self._max_entries:
+            self._cull()
+        self._cache[key] = (value, self.get_backend_timeout(timeout))
+        self._cache.move_to_end(key, last=False)
+
+    def _delete(self, key: str) -> None:
+        try:
+            del self._cache[key]
+        except KeyError:
+            pass
+
+    def _cull(self) -> None:
+        if self._cull_frequency == 0:
+            self._cache.clear()
+        else:
+            count = len(self._cache) // self._cull_frequency
+            for _ in range(count):
+                self._cache.popitem()

shared_memory_dict/templates.py

@@ -1 +1 @@
-MEMORY_NAME = 'tmp/smc_{name}'
+MEMORY_NAME = 'sm_{name}'