Parallel processing of cloning operation

- Setup SQS + Shoryuken worker
- Test for new asynch processing and completion HTTP statuses
- Refactor controller response representation with represent_response method
- Introduce GitRepo as its own datamapper
- Refactor Blame domain objects around GitRepo mapper

Gemfile

@@ -9,6 +9,10 @@ gem 'http'
 # Asynchronicity gems
 gem 'concurrent-ruby'
 
+# Worker gems
+gem 'aws-sdk-sqs', '~> 1'
+gem 'shoryuken', '~> 3'
+
 # Web app related
 gem 'econfig'
 gem 'pry' # to run console in production

Gemfile.lock

@@ -4,6 +4,15 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     ast (2.3.0)
+    aws-partitions (1.44.0)
+    aws-sdk-core (3.11.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -79,6 +88,7 @@ GEM
     http_parser.rb (0.6.0)
     ice_nine (0.11.2)
     inflecto (0.0.2)
+    jmespath (1.3.1)
     json (2.1.0)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
@@ -137,12 +147,17 @@ GEM
     safe_yaml (1.0.4)
     sequel (5.1.0)
     sexp_processor (4.10.0)
+    shoryuken (3.1.12)
+      aws-sdk-core (>= 2)
+      concurrent-ruby
+      thor
     simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     sqlite3 (1.3.13)
+    thor (0.20.0)
     thread_safe (0.3.6)
     uber (0.1.0)
     unf (0.1.4)
@@ -165,6 +180,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk-sqs (~> 1)
   concurrent-ruby
   database_cleaner
   dry-monads
@@ -189,6 +205,7 @@ DEPENDENCIES
   roda
   rubocop
   sequel
+  shoryuken (~> 3)
   simplecov
   sqlite3
   vcr

Procfile

@@ -1 +1,2 @@
-web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
+web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
+worker: bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken.yml

Rakefile

@@ -10,6 +10,7 @@ end
 task :config do
   require_relative 'config/environment.rb' # load config info
   @app = CodePraise::Api
+  @config = @app.config
 end
 
 desc 'run tests'
@@ -20,7 +21,8 @@ end
 
 desc 'rerun tests'
 task :respec => :config do
-  sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'"
+  puts 'REMEMBER: need to run `rake run:[dev|test]:worker` in another process'
+  sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'"
 end
 
 desc 'run application console (pry)'
@@ -29,19 +31,33 @@ task :console do
 end
 
 namespace :run do
-  task :dev => :config do
-    sh "rerun -c 'RACK_ENV=test rackup -p 3030' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'"
+  namespace :api do
+    task :dev => :config do
+      puts 'REMEMBER: need to run `rake run:dev:worker` in another process'
+      sh "rerun -c 'rackup -p 3030' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'"
+    end
+
+    task :test => :config do
+      puts 'REMEMBER: need to run `rake run:test:worker` in another process'
+      sh "rerun -c 'RACK_ENV=test rackup -p 3000' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'"
+    end
   end
 
-  task :test => :config do
-    sh "rerun -c 'RACK_ENV=test rackup -p 3000' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'"
+  namespace :worker do
+    task :dev => :config do
+      sh 'bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken.yml'
+    end
+
+    task :test => :config do
+      sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken_test.yml'
+    end
   end
 end
 
 namespace :ls do
   desc 'list cloned repos in repo store'
   task :repostore => :config do
-    `ls #{@app.config.REPOSTORE_PATH}`
+    puts `ls #{@config.REPOSTORE_PATH}`
   end
 end
 
@@ -55,7 +71,7 @@ namespace :rm do
 
   desc 'delete cloned repos in repo store'
   task :repostore => :config do
-    sh "rm -rf #{@app.config.REPOSTORE_PATH}/*" do |ok, _|
+    sh "rm -rf #{@config.REPOSTORE_PATH}/*" do |ok, _|
       puts(ok ? 'Cloned repos deleted' : "Could not delete cloned repos")
     end
   end
@@ -117,3 +133,42 @@ namespace :db do
     puts "Deleted #{app.config.DB_FILENAME}"
   end
 end
+
+namespace :queue do
+  require 'aws-sdk-sqs'
+
+  desc "Create SQS queue for Shoryuken"
+  task :create => :config do
+    sqs = Aws::SQS::Client.new(region: @config.AWS_REGION)
+
+    begin
+      queue = sqs.create_queue(
+        queue_name: @config.CLONE_QUEUE,
+        attributes: {
+          FifoQueue: 'true',
+          ContentBasedDeduplication: 'true'
+        }
+      )
+
+      q_url = sqs.get_queue_url(queue_name: @config.CLONE_QUEUE)
+      puts "Queue created:"
+      puts "Name: #{@config.CLONE_QUEUE}"
+      puts "Region: #{@config.AWS_REGION}"
+      puts "URL: #{q_url.queue_url}"
+      puts "Environment: #{@app.environment}"
+    rescue => e
+      puts "Error creating queue: #{e}"
+    end
+  end
+
+  task :purge => :config do
+    sqs = Aws::SQS::Client.new(region: @config.AWS_REGION)
+
+    begin
+      queue = sqs.purge_queue(queue_url: @config.CLONE_QUEUE_URL)
+      puts "Queue #{@config.CLONE_QUEUE} purged"
+    rescue => e
+      puts "Error purging queue: #{e}"
+    end
+  end
+end

application/controllers/app.rb

@@ -12,6 +12,17 @@ class Api < Roda
     require_relative 'repo'
     require_relative 'summary'
 
+    def represent_response(result, representer_class)
+      http_response = HttpResponseRepresenter.new(result.value)
+      response.status = http_response.http_code
+      if result.success?
+        yield if block_given?
+        representer_class.new(result.value.message).to_json
+      else
+        http_response.to_json
+      end
+    end
+
     route do |routing|
       response['Content-Type'] = 'application/json'
 

application/controllers/repo.rb

@@ -34,30 +34,19 @@ class Api < Roda
             ownername: ownername, reponame: reponame
           )
 
-          http_response = HttpResponseRepresenter.new(find_result.value)
-          response.status = http_response.http_code
-          if find_result.success?
-            RepoRepresenter.new(find_result.value.message).to_json
-          else
-            http_response.to_json
-          end
+          represent_response(find_result, RepoRepresenter)
         end
 
         # POST #{API_ROOT}/repo/:ownername/:reponame request
         routing.post do
-          service_result = LoadFromGithub.new.call(
+          load_result = LoadFromGithub.new.call(
             config: Api.config,
             ownername: ownername,
             reponame: reponame
           )
 
-          http_response = HttpResponseRepresenter.new(service_result.value)
-          response.status = http_response.http_code
-          if service_result.success?
+          represent_response(load_result, RepoRepresenter) do
             response['Location'] = "#{@api_root}/repo/#{ownername}/#{reponame}"
-            RepoRepresenter.new(service_result.value.message).to_json
-          else
-            http_response.to_json
           end
         end
       end

application/controllers/summary.rb

@@ -17,10 +17,17 @@ class Api < Roda
         routing.get do
           path = request.remaining_path
           folder = path.empty? ? '' : path[1..-1]
-          folder_summary = Blame::Summary.new(@repo).for_folder(folder)
 
-          response.status = 200
-          FolderSummaryRepresenter.new(folder_summary).to_json
+          request_unique = [request.env, request.path, Time.now]
+          request_id = (request_unique.map(&:to_s).join).hash
+
+          summarize_result = SummarizeFolder.new.call(
+            repo: @repo,
+            folder: folder,
+            unique_id: request_id
+          )
+
+          represent_response(summarize_result, FolderSummaryRepresenter)
         end
       end
     end

application/representers/repo_representer.rb

@@ -8,9 +8,10 @@ class RepoRepresenter < Roar::Decorator
     include Roar::JSON
 
     property :origin_id
-    property :owner, extend: CollaboratorRepresenter
+    property :owner, extend: CollaboratorRepresenter, class: OpenStruct
     property :name
     property :git_url
-    collection :contributors, extend: CollaboratorRepresenter
+    property :size
+    collection :contributors, extend: CollaboratorRepresenter, class: OpenStruct
   end
 end

application/services/summarize_folder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'dry/transaction'
+
+module CodePraise
+  # Transaction to summarize folder from local repo
+  class SummarizeFolder
+    include Dry::Transaction
+
+    step :clone_or_find_repo
+    step :summarize_folder
+
+    def clone_or_find_repo(input)
+      input[:gitrepo] = GitRepo.new(input[:repo])
+      if input[:gitrepo].exists_locally?
+        Right(input)
+      else
+        repo_json = RepoRepresenter.new(input[:repo]).to_json
+        CloneRepoWorker.perform_async(repo_json)
+        Left(Result.new(:processing, 'Processing the summary request'))
+      end
+    rescue
+      Left(Result.new(:internal_error, 'Could not clone repo'))
+    end
+
+    def summarize_folder(input)
+      folder_summary = Blame::Summary
+                       .new(input[:gitrepo])
+                       .for_folder(input[:folder])
+      Right(Result.new(:ok, folder_summary))
+    rescue
+      Left(Result.new(:internal_error, 'Could not summarize folder'))
+    end
+  end
+end

domain/init.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-folders = %w[values entities
-             database_repositories github_mappers blame_reporter]
+folders = %w[values entities mappers repositories]
 
 folders.each do |folder|
   require_relative "#{folder}/init.rb"

domain/blame_reporter/blame_report.rb → domain/mappers/blame_mappers/blame_report.rb

@@ -6,9 +6,8 @@ module CodePraise
   module Blame
     # Git blame parsing and reporting services
     class Reporter
-      def initialize(repo, config = CodePraise::Api.config)
-        origin = Git::RemoteRepo.new(repo.git_url)
-        @local = Git::LocalRepo.new(origin, config.REPOSTORE_PATH)
+      def initialize(gitrepo)
+        @local = gitrepo.local
       end
 
       def folder_report(folder_name)

domain/mappers/blame_mappers/blame_summary.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module CodePraise
+  module Blame
+    # Git blame parsing and reporting services
+    class Summary
+      def initialize(gitrepo)
+        @gitrepo = gitrepo
+      end
+
+      def for_folder(folder_name)
+        blame_reports = Blame::Reporter.new(@gitrepo).folder_report(folder_name)
+        Entity::FolderSummary.new(@repo, folder_name, blame_reports)
+      end
+    end
+  end
+end