diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c289bfd6f..0b4ee512d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,16 +14,18 @@ jobs: - python-version: '3.8' target: 'qa' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Start Redis - uses: supercharge/redis-github-action@1.1.0 + uses: supercharge/redis-github-action@1.5.0 - name: Install system dependencies - run: sudo apt-get install libgraphicsmagick1-dev graphicsmagick libjpeg62 zlib1g-dev + run: | + sudo apt-get update + sudo apt-get install libgraphicsmagick1-dev graphicsmagick libjpeg62 zlib1g-dev - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -33,7 +35,7 @@ jobs: echo "::set-output name=dir::$(pip cache dir)" - name: Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: @@ -53,6 +55,6 @@ jobs: TARGET: ${{ matrix.target }} - name: Upload coverage - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: name: Python ${{ matrix.python-version }} diff --git a/docs/management.rst b/docs/management.rst index 26fe4e718..74cb7ab17 100644 --- a/docs/management.rst +++ b/docs/management.rst @@ -29,6 +29,17 @@ useful if your Key Value Store has garbage data not dealt with by cleanup or you're switching Key Value Store backend. +.. _thumbnail-cleanup-delete-timeout: + +thumbnail cleanup_delete_timeout +================================= +``python manage.py thumbnail cleanup_delete_timeout`` + +Deletes thumbnails in the cache, kvstore (database) and storage (filesystem), +if the file's created time is before THUMBNAIL_CLEANUP_DELETE_TIMEOUT seconds ago. +No action will be taken if THUMBNAIL_CLEANUP_DELETE_TIMEOUT is as ``None`` + + .. _thumbnail-clear-delete-referenced: thumbnail clear_delete_referenced diff --git a/docs/reference/settings.rst b/docs/reference/settings.rst index 7d34c2f12..2f18732ae 100644 --- a/docs/reference/settings.rst +++ b/docs/reference/settings.rst @@ -236,6 +236,15 @@ at maximum or ``None`` if your caching backend can handle that as infinite. Only applicable for the Cached DB Key Value Store. +``THUMBNAIL_CLEANUP_DELETE_TIMEOUT`` +=========================== + +- Default: ``3600 * 24 * 365 * 10`` + +Timeout to deletes thumbnails in the cache, kvstore (database) and storage (filesystem), +based on file's created time. If set as ``None`` then no action will be taken. + + ``THUMBNAIL_CACHE`` =================== diff --git a/sorl/thumbnail/conf/defaults.py b/sorl/thumbnail/conf/defaults.py index f1eb6855b..7af1b9ea9 100644 --- a/sorl/thumbnail/conf/defaults.py +++ b/sorl/thumbnail/conf/defaults.py @@ -53,6 +53,7 @@ # Cache timeout for ``cached_db`` store. You should probably keep this at # maximum or ``0`` if your caching backend can handle that as infinite. THUMBNAIL_CACHE_TIMEOUT = 3600 * 24 * 365 * 10 # 10 years +THUMBNAIL_CLEANUP_DELETE_TIMEOUT = 3600 * 24 * 365 * 10 # 10 years # The cache configuration to use for storing thumbnail data THUMBNAIL_CACHE = 'default' diff --git a/sorl/thumbnail/kvstores/base.py b/sorl/thumbnail/kvstores/base.py index eef411d71..f5e80eab4 100644 --- a/sorl/thumbnail/kvstores/base.py +++ b/sorl/thumbnail/kvstores/base.py @@ -93,8 +93,22 @@ def cleanup(self): Cleans up the key value store. In detail: 1. Deletes all key store references for image_files that do not exist and all key references for its thumbnails *and* their image_files. - 2. Deletes or updates all invalid thumbnail keys + 2. Deletes or updates all invalid thumbnail keys. """ + self._cleanup() + + def cleanup_and_delete_if_created_time_before_dt(self, dt): + """ + Cleans up the key value store. In detail: + 1. Deletes all key store references for image_files that: + - do not exist, or + - created time before ``dt`` + and all key references for its thumbnails *and* their image_files. + 2. Deletes or updates all invalid thumbnail keys. + """ + self._cleanup(delete_if_created_time_before_dt=dt) + + def _cleanup(self, delete_if_created_time_before_dt=None): for key in self._find_keys(identity='image'): image_file = self._get(key) @@ -113,8 +127,20 @@ def cleanup(self): thumbnail_keys_set = set(thumbnail_keys) for thumbnail_key in thumbnail_keys: - if not self._get(thumbnail_key): + thumbnail = self._get(thumbnail_key) + if not thumbnail: thumbnail_keys_set.remove(thumbnail_key) + else: + if delete_if_created_time_before_dt is not None: + try: + created_time = thumbnail.storage.get_created_time(thumbnail.name) + except NotImplementedError: + pass + else: + if created_time < delete_if_created_time_before_dt: + thumbnail_keys_set.remove(thumbnail_key) + self.delete(thumbnail, False) + thumbnail.delete() # delete the actual file thumbnail_keys = list(thumbnail_keys_set) diff --git a/sorl/thumbnail/management/commands/thumbnail.py b/sorl/thumbnail/management/commands/thumbnail.py index f7b7468a2..447a9617a 100644 --- a/sorl/thumbnail/management/commands/thumbnail.py +++ b/sorl/thumbnail/management/commands/thumbnail.py @@ -1,10 +1,20 @@ +from datetime import timedelta + from django.core.management.base import BaseCommand +from django.utils import timezone from sorl.thumbnail import default +from sorl.thumbnail.conf import settings from sorl.thumbnail.images import delete_all_thumbnails -VALID_LABELS = ['cleanup', 'clear', 'clear_delete_referenced', 'clear_delete_all'] +VALID_LABELS = [ + 'cleanup', + 'cleanup_delete_timeout', + 'clear', + 'clear_delete_referenced', + 'clear_delete_all', +] class Command(BaseCommand): @@ -16,6 +26,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('args', choices=VALID_LABELS, nargs=1) + # flake8: noqa: C901 def handle(self, *labels, **options): verbosity = int(options.get('verbosity')) label = labels[0] @@ -31,7 +42,34 @@ def handle(self, *labels, **options): return - if label == 'clear_delete_referenced': + elif label == 'cleanup_delete_timeout' and not settings.THUMBNAIL_CLEANUP_DELETE_TIMEOUT: + self.stdout.write( + "THUMBNAIL_CLEANUP_DELETE_TIMEOUT is empty. No action taken", + ending=' ... ' + ) + return + + elif label == 'cleanup_delete_timeout': + if verbosity >= 1: + self.stdout.write( + """ + Cleanup thumbnails and delete if created time before + THUMBNAIL_CLEANUP_DELETE_TIMEOUT seconds ago + """, + ending=' ... ' + ) + + thumbnail_cache_timeout_dt = timezone.now() - timedelta( + seconds=settings.THUMBNAIL_CLEANUP_DELETE_TIMEOUT + ) + default.kvstore.cleanup_and_delete_if_created_time_before_dt(thumbnail_cache_timeout_dt) + + if verbosity >= 1: + self.stdout.write('[Done]') + + return + + elif label == 'clear_delete_referenced': if verbosity >= 1: self.stdout.write( "Delete all thumbnail files referenced in Key Value Store",