try redis force clean

Gemfile.lock

@@ -6,6 +6,7 @@ PATH
       docker-api
       fugit
       optparse
+      redis
       tzinfo-data
 
 GEM
@@ -38,6 +39,7 @@ GEM
     optparse (0.2.0)
     raabro (1.4.0)
     rake (13.0.6)
+    redis (4.7.1)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)

bin/gitlab-janitor

@@ -35,7 +35,8 @@ GitlabJanitor::Util.setup
   cache_size:         ENV.fetch('CACHE_SIZE', '10G'),
   remove:             ENV.fetch('REMOVE', 'false').to_bool,
   log_level:          ENV.fetch('LOG_LEVEL', ::Logger::INFO),
-  docker_host:        ENV.fetch('DOCKER_HOST', 'unix:///var/run/docker.sock')
+  docker_host:        ENV.fetch('DOCKER_HOST', 'unix:///var/run/docker.sock'),
+  redis_url:          ENV.fetch('REDIS_URL', 'redis://127.0.0.1:6379/0')
 }
 
 parser = OptionParser.new do |o|
@@ -90,8 +91,12 @@ parser = OptionParser.new do |o|
     @opts[:remove] = value.strip.to_bool
   end
 
-  o.on("--docker=#{@opts[:docker_host]}", 'ENV[DOCKER_HOST]'.ljust(25) + 'Docker api endpoint.') do |_url|
-    @opts[:docker_host] = value.strip
+  o.on("--docker=#{@opts[:docker_host]}", 'ENV[DOCKER_HOST]'.ljust(25) + 'Docker api endpoint.') do |url|
+    @opts[:docker_host] = url.strip
+  end
+
+  o.on("--redis=#{@opts[:redis_url]}", 'ENV[REDIS_URL]'.ljust(25) + 'Redis endpoint.') do |url|
+    @opts[:redis_url] = url.strip
   end
 
   o.on('--debug', 'ENV[LOG_LEVEL]'.ljust(25) + 'Verbose logs. ENV values: debug, info, warn, error') do
@@ -123,6 +128,7 @@ volumes = GitlabJanitor::VolumeCleaner.new(
 
 images = GitlabJanitor::ImageCleaner.new(
   image_store: File.expand_path(@opts[:image_store]),
+  redis:       @opts[:redis_url],
   delay:       Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
   deadline:    Fugit::Duration.parse(@opts[:image_deadline]).to_sec
 )

gitlab-janitor.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new 'gitlab-janitor' do |spec|
   spec.add_runtime_dependency 'activesupport', '~> 6.0'
   spec.add_runtime_dependency 'docker-api'
   spec.add_runtime_dependency 'fugit'
+  spec.add_runtime_dependency 'redis'
   spec.add_runtime_dependency 'optparse'
   spec.add_runtime_dependency 'tzinfo-data'
 end

lib/gitlab_janitor/image_cleaner.rb

@@ -1,4 +1,5 @@
 require 'open3'
+require 'redis'
 
 module GitlabJanitor
   class ImageCleaner < BaseCleaner
@@ -41,13 +42,18 @@ def id
 
     attr_reader :store
 
-    def initialize(image_store:, **kwargs)
+    def initialize(image_store:, redis: nil, redis_list: 'gitlab-janitor:images_force_clean', **kwargs)
       super(**kwargs)
       @store = Store.new(filename: image_store, logger: logger)
+      @redis_url = redis
+      @redis_list = redis_list
     end
 
     def do_clean(remove: false)
       store.load
+
+      force_clean(remove: remove)
+
       to_remove, keep = prepare(store.parse_images)
       store.save(skip_older: Time.now - @deadline)
 
@@ -103,6 +109,31 @@ def select_by_deadline(images)
       end
     end
 
+    def force_clean(remove: false)
+      return if @redis_url.nil?
+
+      redis = Redis.new(url: @redis_url)
+      redis.ltrim(@redis_list, 0, 10)
+      now = Time.now
+      redis.lrange(@redis_list, 0, -1).each do |pair|
+        image, ts = pair.split('|')
+        if (now - Time.at(ts.to_i)) > 10.seconds
+          if remove
+            logger.info("Force clean #{image}")
+            log_exception('Remove') { out, _status = Open3.capture2e("docker rmi #{image}"); logger.info(out) }
+          else
+            logger.info("Skip Force clean #{image} due to dry run")
+          end
+        else
+          logger.info("Delay force clean #{image} by time")
+        end
+      rescue StandardError => e
+        logger.error("Error from force line: '#{pair}': #{e.inspect}")
+      end
+    rescue StandardError => e
+      logger.error("Unable to retrieve data from redis: #{e}")
+    end
+
   end
 end