diff --git a/README.rst b/README.rst index b0a8332..fe37597 100644 --- a/README.rst +++ b/README.rst @@ -51,4 +51,19 @@ When ``thumbor_cloud_storage.result_storages.cloud_storage`` is enabled: RESULT_STORAGE = 'thumbor_cloud_storage.result_storages.cloud_storage' RESULT_STORAGE_CLOUD_STORAGE_PROJECT_ID = '' RESULT_STORAGE_CLOUD_STORAGE_BUCKET_ID = '' + RESULT_STORAGE_EXPIRATION_SECONDS = 0 + RESULT_STORAGE_CLOUD_STORAGE_ROOT_PATH='thumbor/result_storage' + +Storage settings +~~~~~~~~~~~~~~~~ + +When ``thumbor_cloud_storage.storages.cloud_storage`` is enabled: + +.. code:: python + + STORAGE = 'thumbor_cloud_storage.storages.cloud_storage' + STORAGE_CLOUD_STORAGE_PROJECT_ID = '' + STORAGE_CLOUD_STORAGE_BUCKET_ID = '' + STORAGE_EXPIRATION_SECONDS = 0 + STORAGE_CLOUD_STORAGE_ROOT_PATH='thumbor/storage' diff --git a/thumbor_cloud_storage/result_storages/cloud_storage.py b/thumbor_cloud_storage/result_storages/cloud_storage.py index b70d939..4c540dc 100644 --- a/thumbor_cloud_storage/result_storages/cloud_storage.py +++ b/thumbor_cloud_storage/result_storages/cloud_storage.py @@ -74,7 +74,7 @@ def get(self): return blob.download_as_string() def normalize_path(self, path): - path_segments = [self.context.config.get('RESULT_STORAGE_CLOUD_STORAGE_ROOT_PATH','thumbor/').rstrip('/'), Storage.PATH_FORMAT_VERSION, ] + path_segments = [self.context.config.get('RESULT_STORAGE_CLOUD_STORAGE_ROOT_PATH','thumbor/result_storage').rstrip('/'), Storage.PATH_FORMAT_VERSION, ] if self.is_auto_webp: path_segments.append("webp") diff --git a/thumbor_cloud_storage/storages/__init__.py b/thumbor_cloud_storage/storages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/thumbor_cloud_storage/storages/cloud_storage.py b/thumbor_cloud_storage/storages/cloud_storage.py new file mode 100644 index 0000000..8f11a3b --- /dev/null +++ b/thumbor_cloud_storage/storages/cloud_storage.py @@ -0,0 +1,125 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# https://github.com/superbalist/thumbor-cloud-storage + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license + +import pytz +from datetime import datetime + +from os.path import join +from gcloud import storage + +from thumbor.result_storages import BaseStorage +from thumbor.utils import logger +from thumbor.engines import BaseEngine + + +class Storage(BaseStorage): + PATH_FORMAT_VERSION = 'v2' + bucket = None + + def __init__(self, context, shared_client=True): + BaseStorage.__init__(self, context) + self.shared_client = shared_client + self.bucket = self.get_bucket() + + def get_bucket(self): + parent = self + if self.shared_client: + parent = Storage + if not parent.bucket: + bucket_id = self.context.config.get("STORAGE_CLOUD_STORAGE_BUCKET_ID") + project_id = self.context.config.get("STORAGE_CLOUD_STORAGE_PROJECT_ID") + client = storage.Client(project_id) + parent.bucket = client.get_bucket(bucket_id) + return parent.bucket + + @property + def is_auto_webp(self): + return self.context.config.AUTO_WEBP and self.context.request.accepts_webp + + def put(self, path, bytes): + file_abspath = self.normalize_path(path) + logger.debug("[STORAGE] putting at %s" % file_abspath) + bucket = self.get_bucket() + + blob = bucket.blob(file_abspath) + blob.upload_from_string(bytes) + + max_age = self.context.config.MAX_AGE + blob.cache_control = "public,max-age=%s" % max_age + + if bytes: + try: + mime = BaseEngine.get_mimetype(bytes) + blob.content_type = mime + except: + logger.debug("[STORAGE] Couldn't determine mimetype") + + + blob.patch() + + def put_crypto(self, path): + ''' + :param path: ignored. always returns None + :return: None, this is not supported + ''' + return + + def get(self, path): + file_abspath = self.normalize_path(path) + logger.debug("[STORAGE] getting from %s" % file_abspath) + + bucket = self.get_bucket() + blob = bucket.get_blob(file_abspath) + if not blob or self.is_expired(blob): + return None + return blob.download_as_string() + + def normalize_path(self, path): + path_segments = [self.context.config.get('STORAGE_CLOUD_STORAGE_ROOT_PATH','thumbor/storage').rstrip('/'), Storage.PATH_FORMAT_VERSION, ] + if self.is_auto_webp: + path_segments.append("webp") + + path_segments.extend([self.partition(path), path.lstrip('/'), ]) + + normalized_path = join(*path_segments).replace('http://', '') + return normalized_path + + def partition(self, path_raw): + path = path_raw.lstrip('/') + return join("".join(path[0:2]), "".join(path[2:4])) + + def is_expired(self, blob): + expire_in_seconds = self.context.config.get('STORAGE_EXPIRATION_SECONDS', None) + + if expire_in_seconds is None or expire_in_seconds == 0: + return False + + timediff = datetime.now(pytz.utc) - blob.updated + return timediff.seconds > expire_in_seconds + + def last_updated(self): + path = self.context.request.url + file_abspath = self.normalize_path(path) + logger.debug("[STORAGE] getting from %s" % file_abspath) + + bucket = self.get_bucket() + blob = bucket.get_blob(file_abspath) + + if not blob or self.is_expired(blob): + logger.debug("[STORAGE] image not found at %s" % file_abspath) + return True + + return blob.updated + + def exists(self, path): + file_abspath = self.normalize_path(path) + logger.debug("[STORAGE] getting from %s" % file_abspath) + + bucket = self.get_bucket() + blob = bucket.get_blob(file_abspath) + return blob and not self.is_expired(blob)