fixes

.rspec

@@ -0,0 +1,6 @@
+--color
+--tty
+--format documentation
+--require spec_helper
+--format RspecJunitFormatter
+--out rspec.xml

Dockerfile

@@ -15,7 +15,7 @@ RUN set -ex \
   && apk add --no-cache docker-cli
 
 ADD Gemfile Gemfile.lock gitlab-janitor.gemspec /home/app/
-ADD lib/gitlab-_anitor/version.rb /home/app/lib/gitlab_janitor/
+ADD lib/gitlab_janitor/version.rb /home/app/lib/gitlab_janitor/
 
 RUN set -ex \
   && gem install bundler && gem update bundler \

README.md

@@ -6,6 +6,13 @@
 
 GitLab Janitor is a tool to automatically manage stalled containers when using Docker.
 
+Features
+
+- Remove dangling containers
+- Remove unused anonymous volumes
+- Remove unused images
+- Track image usage timestamp 
+
 ## Installation
 
 ```sh
@@ -41,15 +48,18 @@ Commain line options and default values:
 $ gitlab-janitor --help
 
 Usage: gitlab-janitor [options] 
-        --clean-delay=30m            Delay between clean operation ENV[CLEAN_DELAY]
-        --include=*units*            <List> Include container for removal. ENV[INCLUDE]
-        --exclude=*gitlab*           <List> Exclude container from removal by name. ENV[EXCLUDE]
-        --container-deadline=1s      Maximum container run duration. ENV[CONTAINER_DEADLINE]
-        --volume-deadline=2d6h       Maximum volume life dudation. ENV[VOLUME_DEADLINE]
-        --image-deadline=20d         Maximum image life duration. ENV[IMAGE_DEADLINE]
-        --remove                     Real remove instead of dry run. ENV[REMOVE]
+        --clean-delay=30m            ENV[CLEAN_DELAY]         Delay between clean operation.
+        --include=*units*            ENV[INCLUDE]             <List> Include container for removal.
+        --exclude=*gitlab*           ENV[EXCLUDE]             <List> Exclude container from removal by name.
+        --container-deadline=1h10m   ENV[CONTAINER_DEADLINE]  Maximum container run duration.
+        --volume-deadline=2d6h       ENV[VOLUME_DEADLINE]     Maximum volume life dudation.
+        --image-deadline=20d         ENV[IMAGE_DEADLINE]      Maximum image life duration.
+        --image-store=./images.txt   ENV[IMAGE_STORE]         File to store images timestamps.
+        --remove                     ENV[REMOVE]              Real remove instead of dry run.
         --docker=unix:///var/run/docker.sock
-                                     Docker api endpoint. ENV[DOCKER_HOST]
+                                     ENV[DOCKER_HOST]         Docker api endpoint.
+        --debug                      ENV[LOG_LEVEL]           Verbose logs. ENV values: debug, info, warn, error
+
 ```
 
 ### Удаление зависших контейнеров
@@ -73,10 +83,11 @@ Docker не сохраняет временную метку образа при
 
 Порядок определения образов для удалени:
 
-- на удаление попадают только образы, не имеющие тэг `latest`;
+- При первой встрече нового образа врменная метка сохраняется в локальное хранилище (файл)
+- При достижении лимита хранения образ удаляется
+- При обнаружении запущеннго контейнера временная метка для соответствующего образа обнуляется
 - `image-deadline=[20d]` - результирующий список проверяется на длительность существования образа;
 
-
 ## Examples
 
 Production ready config:

bin/gitlab-janitor

@@ -28,79 +28,100 @@ GitlabJanitor::Util.setup
   excludes:           [ENV.fetch('EXCLUDE', '*gitlab*')],
   clean_delay:        ENV.fetch('CLEAN_DELAY', '30m'),
   container_deadline: ENV.fetch('CONTAINER_DEADLINE', '1h10m'),
+  volume_includes:    [ENV.fetch('IMAGE_INCLUDE', 'runner*cache*')],
   volume_deadline:    ENV.fetch('VOLUME_DEADLINE', '2d6h'),
   image_deadline:     ENV.fetch('IMAGE_DEADLINE', '20d'),
+  image_store:        ENV.fetch('IMAGE_STORE', './images.txt'),
   remove:             ENV.fetch('REMOVE', 'false').to_bool,
+  log_level:          ENV.fetch('LOG_LEVEL', ::Logger::INFO),
   docker_host:        ENV.fetch('DOCKER_HOST', 'unix:///var/run/docker.sock')
 }
 
 parser = OptionParser.new do |o|
   o.banner = "Usage: #{UTIL} [options] "
 
   o.on("--clean-delay=#{@opts[:clean_delay]}",
-       'Delay between clean operation ENV[CLEAN_DELAY]') do |pattern|
+       'ENV[CLEAN_DELAY]'.ljust(25) + 'Delay between clean operation.') do |pattern|
     @opts[:clean_delay] = pattern.strip
   end
 
   o.on("--include=#{@opts[:includes].join(',')}",
-       '<List> Include container for removal. ENV[INCLUDE]') do |pattern|
+       'ENV[INCLUDE]'.ljust(25) + '<List> Include container for removal.') do |pattern|
     @opts[:includes] += pattern.split(/[,;]/)
   end
 
   o.on("--exclude=#{@opts[:excludes].join(',')}",
-       '<List> Exclude container from removal by name. ENV[EXCLUDE]') do |pattern|
+       'ENV[EXCLUDE]'.ljust(25) + '<List> Exclude container from removal by name.') do |pattern|
     @opts[:excludes] += pattern.split(/[,;]/)
   end
 
   o.on("--container-deadline=#{@opts[:container_deadline]}",
-       'Maximum container run duration. ENV[CONTAINER_DEADLINE]') do |pattern|
+       'ENV[CONTAINER_DEADLINE]'.ljust(25) + 'Maximum container run duration.') do |pattern|
     @opts[:container_deadline] = pattern.strip
   end
 
+  o.on("--volume-include=#{@opts[:volume_includes].join(',')}",
+       'ENV[VOLUME_INCLUDE]'.ljust(25) + '<List> Include volumes for removal.') do |pattern|
+    @opts[:volume_includes] += pattern.split(/[,;]/)
+  end
+
   o.on("--volume-deadline=#{@opts[:volume_deadline]}",
-       'Maximum volume life dudation. ENV[VOLUME_DEADLINE]') do |pattern|
+       'ENV[VOLUME_DEADLINE]'.ljust(25) + 'Maximum volume life duration.') do |pattern|
     @opts[:volume_deadline] = pattern.strip
   end
 
   o.on("--image-deadline=#{@opts[:image_deadline]}",
-       'Maximum image life duration. ENV[IMAGE_DEADLINE]') do |pattern|
+       'ENV[IMAGE_DEADLINE]'.ljust(25) + 'Maximum image life duration.') do |pattern|
     @opts[:image_deadline] = pattern.strip
   end
 
-  o.on('--remove', 'Real remove instead of dry run. ENV[REMOVE]') do |value|
+  o.on("--image-store=#{@opts[:image_store]}",
+       'ENV[IMAGE_STORE]'.ljust(25) + 'File to store images timestamps.') do |pattern|
+    @opts[:image_store] = pattern.strip
+  end
+
+  o.on('--remove', 'ENV[REMOVE]'.ljust(25) + 'Real remove instead of dry run.') do |value|
     @opts[:remove] = value.strip.to_bool
   end
 
-  o.on("--docker=#{@opts[:docker_host]}", 'Docker api endpoint. ENV[DOCKER_HOST]') do |_url|
+  o.on("--docker=#{@opts[:docker_host]}", 'ENV[DOCKER_HOST]'.ljust(25) + 'Docker api endpoint.') do |_url|
     @opts[:docker_host] = value.strip
   end
+
+  o.on('--debug', 'ENV[LOG_LEVEL]'.ljust(25) + 'Verbose logs. ENV values: debug, info, warn, error') do
+    @opts[:log_level] = ::Logger::DEBUG
+  end
 end
 parser.parse!
 
 Docker.url = @opts[:docker_host]
 
+GitlabJanitor::Util.logger.level = @opts[:log_level]
+
 GitlabJanitor::Util.logger.debug do
   "Config: #{JSON.pretty_generate(@opts)}"
 end
 
 containers = GitlabJanitor::ContainerCleaner.new(
-  delay:    Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
   includes: @opts[:includes],
   excludes: @opts[:excludes],
+  delay:    Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
   deadline: Fugit::Duration.parse(@opts[:container_deadline]).to_sec
 )
 
 volumes = GitlabJanitor::VolumeCleaner.new(
+  includes: @opts[:volume_includes],
   delay:    Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
   deadline: Fugit::Duration.parse(@opts[:volume_deadline]).to_sec
 )
 
 images = GitlabJanitor::ImageCleaner.new(
-  delay:    Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
-  deadline: Fugit::Duration.parse(@opts[:image_deadline]).to_sec
+  image_store: File.expand_path(@opts[:image_store]),
+  delay:       Fugit::Duration.parse(@opts[:clean_delay]).to_sec,
+  deadline:    Fugit::Duration.parse(@opts[:image_deadline]).to_sec
 )
 
-until GitlabJanitor.exiting?
+until GitlabJanitor::Util.exiting?
   containers.clean(remove: @opts[:remove])
   volumes.clean(remove: @opts[:remove])
   images.clean(remove: @opts[:remove])

lib/gitlab_janitor/base_cleaner.rb

@@ -42,6 +42,10 @@ def clean(remove: false)
       true
     end
 
+    def exiting?
+      GitlabJanitor::Util.exiting?
+    end
+
     def log_exception(text)
       yield
     rescue StandardError => e

lib/gitlab_janitor/container_cleaner.rb

@@ -29,8 +29,8 @@ def age_text
 
     attr_reader :excludes, :includes
 
-    def initialize(includes: [''], excludes: [''], **args)
-      super(**args)
+    def initialize(includes: [''], excludes: [''], **kwargs)
+      super(**kwargs)
       @includes = includes
       @excludes = excludes
       @deadline = deadline
@@ -45,12 +45,14 @@ def do_clean(remove: false)
 
       if remove
         logger.info 'Removing containers...'
-        to_remove.each do |c|
-          logger.tagged(c.name) do
+        to_remove.each do |model|
+          return false if exiting?
+
+          logger.tagged(model.name) do
             logger.debug '   Removing...'
-            log_exception('Stop') { c.stop }
-            log_exception('Wait') { c.wait(15) }
-            log_exception('Remove') { c.remove }
+            log_exception('Stop') { model.stop }
+            log_exception('Wait') { model.wait(15) }
+            log_exception('Remove') { model.remove }
             logger.debug '   Removing COMPLETED'
           end
         end

lib/gitlab_janitor/image_cleaner.rb

@@ -39,26 +39,30 @@ def id
 
     require_relative 'image_cleaner/store'
 
-    def initialize(**kwargs)
-      super
-      @store = Store.new(logger: logger)
+    attr_reader :store
+
+    def initialize(image_store:, **kwargs)
+      super(**kwargs)
+      @store = Store.new(filename: image_store, logger: logger)
     end
 
     def do_clean(remove: false)
-      @store.load
-      to_remove, keep = prepare(@store.parse_images)
-      @store.save(skip_older: Time.now - @deadline)
+      store.load
+      to_remove, keep = prepare(store.parse_images)
+      store.save(skip_older: Time.now - @deadline)
 
       return if to_remove.empty?
 
       keep.each {|m| logger.debug("  KEEP #{m.name}") }
 
       if remove
         logger.info 'Removing images...'
-        to_remove.each do |c|
-          logger.tagged(c.name) do
+        to_remove.each do |model|
+          return false if exiting?
+
+          logger.tagged(model.name) do
             logger.debug '   Removing...'
-            log_exception('Remove') { `docker rmi #{c.name}` }
+            log_exception('Remove') { `docker rmi #{model.name}` }
             logger.debug '   Removing COMPLETED'
           end
         end
@@ -69,6 +73,7 @@ def do_clean(remove: false)
 
     def prepare(images)
       to_remove = images
+
       @logger.info("Selected images: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
 
       @logger.debug("Filtering images by deadline: older than #{Fugit::Duration.parse(@deadline).deflate.to_plain_s}...")
@@ -86,8 +91,8 @@ def format_item(model)
       "#{model.loaded_at} Age:#{model.age_text.ljust(13)} #{model.name.first(60).ljust(60)}"
     end
 
-    def select_by_deadline(containers)
-      containers.select do |model|
+    def select_by_deadline(images)
+      images.select do |model|
         model.age > deadline
       end
     end

lib/gitlab_janitor/volume_cleaner.rb

@@ -31,6 +31,13 @@ def mountpoint
 
     end
 
+    attr_reader :includes
+
+    def initialize(includes: [''], **kwargs)
+      super(**kwargs)
+      @includes = includes
+    end
+
     def do_clean(remove: false)
       to_remove, keep = prepare(Docker::Volume.all.map{|m| Model.new(m) })
 
@@ -41,6 +48,8 @@ def do_clean(remove: false)
       if remove
         logger.info 'Removing volumes...'
         to_remove.each do |model|
+          return false if exiting?
+
           logger.tagged(model.name.first(10)) do
             logger.debug '   Removing...'
             log_exception('Remove') { model.remove }
@@ -61,6 +70,14 @@ def prepare(volumes)
       end
       @logger.info("Selected volumes: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
 
+      @logger.debug("Selecting volumes by includes #{@includes}...")
+      to_remove += select_by_name(volumes)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], images
+      end
+      @logger.info("Selected volumes: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
       @logger.debug("Filtering volumes by deadline: older than #{@deadline} seconds...")
       to_remove = select_by_deadline(to_remove)
       if to_remove.empty?
@@ -76,6 +93,14 @@ def format_item(model)
       "#{Time.parse(model.created_at)} Age:#{model.age_text.ljust(13)} #{model.name.first(10).ljust(10)} #{model.mountpoint}"
     end
 
+    def select_by_name(volumes)
+      volumes.select do |model|
+        @includes.any? do |pattern|
+          File.fnmatch(pattern, model.name)
+        end
+      end
+    end
+
     SHA_RX = /^[a-zA-Z0-9]{64}$/.freeze
 
     def select_unnamed(volumes)
@@ -84,8 +109,8 @@ def select_unnamed(volumes)
       end
     end
 
-    def select_by_deadline(containers)
-      containers.select do |model|
+    def select_by_deadline(volumes)
+      volumes.select do |model|
         model.age > deadline
       end
     end

spec/support/log_helper.rb

@@ -1,4 +1,3 @@
-
 RSpec.configure do |config|
   config.before(:suite) do
     GitlabJanitor::Util.logger.level = ::Logger::WARN