diff --git a/Gemfile b/Gemfile index d9d9cf5..a6b4451 100644 --- a/Gemfile +++ b/Gemfile @@ -9,8 +9,9 @@ gem 'http' # Asynchronicity gems gem 'concurrent-ruby' -# Worker gems +# Parallel worker gem 'aws-sdk-sqs', '~> 1' +gem 'faye', '~> 1' gem 'shoryuken', '~> 3' # Web app related diff --git a/Gemfile.lock b/Gemfile.lock index 9688df0..5b0381c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,6 +23,7 @@ GEM coercible (1.0.0) descendants_tracker (~> 0.0.1) concurrent-ruby (1.0.5) + cookiejar (0.3.3) crack (0.4.3) safe_yaml (~> 1.0.0) database_cleaner (1.6.1) @@ -69,7 +70,27 @@ GEM dry-logic (~> 0.4, >= 0.4.2) inflecto (~> 0.0.0, >= 0.0.2) econfig (2.0.0) + em-http-request (1.1.5) + addressable (>= 2.3.4) + cookiejar (!= 0.3.1) + em-socksify (>= 0.3) + eventmachine (>= 1.0.3) + http_parser.rb (>= 0.6.0) + em-socksify (0.3.1) + eventmachine (>= 1.0.0.beta.4) equalizer (0.0.11) + eventmachine (1.2.5) + faye (1.2.4) + cookiejar (>= 0.3.0) + em-http-request (>= 0.3.0) + eventmachine (>= 0.12.0) + faye-websocket (>= 0.9.1) + multi_json (>= 1.0.0) + rack (>= 1.0.0) + websocket-driver (>= 0.5.1) + faye-websocket (0.10.7) + eventmachine (>= 0.12.0) + websocket-driver (>= 0.5.1) ffi (1.9.18) flog (4.6.1) path_expander (~> 1.0) @@ -174,6 +195,9 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff + websocket-driver (0.7.0) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.3) wisper (2.0.0) PLATFORMS @@ -188,6 +212,7 @@ DEPENDENCIES dry-transaction dry-types econfig + faye (~> 1) flog hirb http diff --git a/Rakefile b/Rakefile index 6d279ec..3fda8e3 100644 --- a/Rakefile +++ b/Rakefile @@ -35,15 +35,21 @@ namespace :api do namespace :run do desc 'Rerun the API server in development mode' task :development => :config do - puts 'REMEMBER: need to run `rake run:dev:worker` in another process' + puts 'REMEMBER: need to run `rake worker:run:development` in another process' sh "rerun -c 'rackup -p 3030' --ignore '#{@config.REPOSTORE_PATH}/*'" end desc 'Rerun the API server in test mode' task :test => :config do - puts 'REMEMBER: need to run `rake run:test:worker` in another process' + puts 'REMEMBER: need to run `rake worker:run:test` in another process' sh "rerun -c 'RACK_ENV=test rackup -p 3000' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'" end + + desc 'Run the API server to test the client app' + task :app_test => :config do + puts 'REMEMBER: need to run `rake worker:run:app_test` in another process' + sh 'RACK_ENV=test rackup -p 3000' + end end end @@ -59,6 +65,11 @@ namespace :worker do sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken_test.yml' end + desc 'Run the background cloning worker in testing mode' + task :app_test => :config do + sh 'RACK_ENV=app_test bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken_test.yml' + end + desc 'Run the background cloning worker in production mode' task :production => :config do sh 'RACK_ENV=production bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken.yml' diff --git a/application/controllers/app.rb b/application/controllers/app.rb index d21bb7d..6a936cf 100644 --- a/application/controllers/app.rb +++ b/application/controllers/app.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'roda' -# require_relative 'routes/repo' +require_relative 'route_helpers' module CodePraise # Web API @@ -9,19 +9,8 @@ class Api < Roda plugin :all_verbs plugin :multi_route - require_relative 'repo' - require_relative 'summary' - - def represent_response(result, representer_class) - http_response = HttpResponseRepresenter.new(result.value) - response.status = http_response.http_code - if result.success? - yield if block_given? - representer_class.new(result.value.message).to_json - else - http_response.to_json - end - end + require_relative 'repo_controller' + require_relative 'summary_controller' route do |routing| response['Content-Type'] = 'application/json' diff --git a/application/controllers/repo.rb b/application/controllers/repo_controller.rb similarity index 100% rename from application/controllers/repo.rb rename to application/controllers/repo_controller.rb diff --git a/application/controllers/route_helpers.rb b/application/controllers/route_helpers.rb new file mode 100644 index 0000000..6c88d02 --- /dev/null +++ b/application/controllers/route_helpers.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module CodePraise + class Api < Roda + # Represent HTTP response for result + # Parameters: + # - result: Result object with #message to represent + # - success_representer: representer class if result is success + # #to_json called if result is failure + # - (optional) block to execute before success representation + # Returns: Json representation of success/failure message + def represent_response(result, success_representer) + http_response = HttpResponseRepresenter.new(result.value) + response.status = http_response.http_code + if result.success? + yield if block_given? + success_representer.new(result.value.message).to_json + else + http_response.to_json + end + end + + # Extracts sub-resource path from request + # Parameters: HTTP request (Roda request object) + # Returns: folder path (string) + def folder_name_from(request) + path = request.remaining_path + path.empty? ? '' : path[1..-1] + end + end +end diff --git a/application/controllers/summary.rb b/application/controllers/summary_controller.rb similarity index 50% rename from application/controllers/summary.rb rename to application/controllers/summary_controller.rb index 92310f2..a1656c8 100644 --- a/application/controllers/summary.rb +++ b/application/controllers/summary_controller.rb @@ -14,22 +14,21 @@ class Api < Roda routing.halt(404, 'Repo not found') if find_result.failure? @repo = find_result.value.message - routing.get do - path = request.remaining_path - folder = path.empty? ? '' : path[1..-1] + routing.get do + path = request.remaining_path + folder = path.empty? ? '' : path[1..-1] - request_unique = [request.env, request.path, Time.now] - request_id = (request_unique.map(&:to_s).join).hash + request_id = [request.env, request.path, Time.now.to_f].hash - summarize_result = SummarizeFolder.new.call( - repo: @repo, - folder: folder, - unique_id: request_id - ) + summarize_result = SummarizeFolder.new.call( + repo: @repo, + folder: folder, + id: request_id + ) - represent_response(summarize_result, FolderSummaryRepresenter) - end + represent_response(summarize_result, FolderSummaryRepresenter) + end end end end -end \ No newline at end of file +end diff --git a/application/representers/clone_request_representer.rb b/application/representers/clone_request_representer.rb new file mode 100644 index 0000000..ab5cccc --- /dev/null +++ b/application/representers/clone_request_representer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require_relative 'collaborator_representer' + +# Represents essential Repo information for API output +module CodePraise + class CloneRequestRepresenter < Roar::Decorator + include Roar::JSON + + property :repo, extend: RepoRepresenter, class: OpenStruct + property :id + end +end diff --git a/application/services/summarize_folder.rb b/application/services/summarize_folder.rb index 1b53453..002cb7f 100644 --- a/application/services/summarize_folder.rb +++ b/application/services/summarize_folder.rb @@ -7,19 +7,25 @@ module CodePraise class SummarizeFolder include Dry::Transaction - step :clone_or_find_repo + step :find_repo + step :clone_repo step :summarize_folder - def clone_or_find_repo(input) + def find_repo(input) input[:gitrepo] = GitRepo.new(input[:repo]) + Right(input) + end + + def clone_repo(input) if input[:gitrepo].exists_locally? Right(input) else - repo_json = RepoRepresenter.new(input[:repo]).to_json - CloneRepoWorker.perform_async(repo_json) - Left(Result.new(:processing, 'Processing the summary request')) + clone_request = clone_request_json(input) + CloneRepoWorker.perform_async(clone_request.to_json) + Left(Result.new(:processing, { id: input[:id] })) end - rescue + rescue StandardError => error + puts "ERROR: SummarizeFolder#clone_repo - #{error.inspect}" Left(Result.new(:internal_error, 'Could not clone repo')) end @@ -28,8 +34,15 @@ def summarize_folder(input) .new(input[:gitrepo]) .for_folder(input[:folder]) Right(Result.new(:ok, folder_summary)) - rescue + rescue StandardError Left(Result.new(:internal_error, 'Could not summarize folder')) end + + private + + def clone_request_json(input) + clone_request = CloneRequest.new(input[:repo], input[:id]) + CloneRequestRepresenter.new(clone_request) + end end -end \ No newline at end of file +end diff --git a/config.ru b/config.ru index a49bf64..68f6ca8 100644 --- a/config.ru +++ b/config.ru @@ -1,4 +1,7 @@ # frozen_string_literal: true +require 'faye' require_relative './init.rb' + +use Faye::RackAdapter, :mount => '/faye', :timeout => 25 run CodePraise::Api.freeze.app diff --git a/config/app.yml b/config/app.yml index ea145c5..0b7eb4c 100644 --- a/config/app.yml +++ b/config/app.yml @@ -1,8 +1,19 @@ --- development: + API_URL: 'http://localhost:3030' DB_FILENAME: infrastructure/database/dev.db REPOSTORE_PATH: infrastructure/gitrepo/repostore test: + API_URL: 'http://localhost:3000' DB_FILENAME: infrastructure/database/test.db + REPOSTORE_PATH: infrastructure/gitrepo/repostore + +app_test: + API_URL: 'http://localhost:3000' + DB_FILENAME: infrastructure/database/test.db + REPOSTORE_PATH: infrastructure/gitrepo/repostore + +production: + API_URL: 'https://codepraise-api.herokuapp.com' REPOSTORE_PATH: infrastructure/gitrepo/repostore \ No newline at end of file diff --git a/domain/mappers/git_mappers/git_repo.rb b/domain/mappers/git_mappers/git_repo.rb index 8efae36..1404562 100644 --- a/domain/mappers/git_mappers/git_repo.rb +++ b/domain/mappers/git_mappers/git_repo.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module CodePraise + # Maps over local and remote git repo infrastructure class GitRepo MAX_SIZE = 1000 # for cloning, analysis, summaries, etc. @@ -12,8 +13,8 @@ class Errors def initialize(repo, config = CodePraise::Api.config) @repo = repo - origin = Git::RemoteRepo.new(@repo.git_url) - @local = Git::LocalRepo.new(origin, config.REPOSTORE_PATH) + remote = Git::RemoteRepo.new(@repo.git_url) + @local = Git::LocalRepo.new(remote, config.REPOSTORE_PATH) end def local @@ -36,7 +37,7 @@ def exists_locally? def clone! raise Errors::TooLargeToClone if too_large? raise Errors::CannotOverwriteLocalRepo if exists_locally? - @local.clone_remote + @local.clone_remote { |line| yield line if block_given? } end end end diff --git a/domain/values/clone_request.rb b/domain/values/clone_request.rb new file mode 100644 index 0000000..ef4ac89 --- /dev/null +++ b/domain/values/clone_request.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module CodePraise + CloneRequest = Struct.new :repo, :id +end diff --git a/infrastructure/gitrepo/local_repo.rb b/infrastructure/gitrepo/local_repo.rb index 7e54af1..ed59511 100644 --- a/infrastructure/gitrepo/local_repo.rb +++ b/infrastructure/gitrepo/local_repo.rb @@ -23,7 +23,7 @@ def initialize(remote, repostore_path) end def clone_remote - @remote.local_clone(@repo_path) + @remote.local_clone(@repo_path) { |line| yield line if block_given? } self end @@ -78,4 +78,4 @@ def wipe FileUtils.rm_rf @repo_path end end -end \ No newline at end of file +end diff --git a/infrastructure/gitrepo/remote_repo.rb b/infrastructure/gitrepo/remote_repo.rb index 101f685..16d11af 100644 --- a/infrastructure/gitrepo/remote_repo.rb +++ b/infrastructure/gitrepo/remote_repo.rb @@ -16,19 +16,16 @@ def initialize(git_url) @git_url = git_url end - def local_clone(path) - `git clone --progress #{@git_url} #{path} 2>&1` - - # Cloning into 'infrastructure/gitrepo/repostore/test_cmdline'... - # remote: Counting objects: 860, done. - # remote: Total 860 (delta 0), reused 0 (delta 0), pack-reused 860 - # Receiving objects: 100% (860/860), 543.83 KiB | 0 bytes/s, done. - # Resolving deltas: 100% (516/516), done. - # Checking connectivity... done. - end - def unique_id Base64.urlsafe_encode64(Digest::SHA256.digest(@git_url)) end + + def local_clone(path) + command = "git clone --progress #{@git_url} #{path} 2>&1" + + IO.popen(command).each do |line| + yield line if block_given? + end + end end end diff --git a/workers/clone_repo_worker.rb b/workers/clone_repo_worker.rb index 38f812b..21c2e35 100644 --- a/workers/clone_repo_worker.rb +++ b/workers/clone_repo_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'load_all' - +require 'http' require 'econfig' require 'shoryuken' @@ -11,6 +11,8 @@ class CloneRepoWorker Econfig.env = ENV['RACK_ENV'] || 'development' Econfig.root = File.expand_path('..', File.dirname(__FILE__)) + require_relative 'test_helper' if ENV['RACK_ENV'] == 'test' + Shoryuken.sqs_client = Aws::SQS::Client.new( access_key_id: config.AWS_ACCESS_KEY_ID, secret_access_key: config.AWS_SECRET_ACCESS_KEY, @@ -20,13 +22,48 @@ class CloneRepoWorker include Shoryuken::Worker shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true - def perform(_sqs_msg, worker_request) - request = CodePraise::RepoRepresenter.new(OpenStruct.new).from_json worker_request - puts "REQUEST: #{request}" - gitrepo = CodePraise::GitRepo.new(request) - puts "EXISTS: #{gitrepo.exists_locally?}" - gitrepo.clone! - puts "REQUEST: #{request}" - puts "EXISTS: #{gitrepo.exists_locally?}" + def perform(_sqs_msg, request_json) + clone_request = CodePraise::CloneRequestRepresenter + .new(CodePraise::CloneRequest.new) + .from_json(request_json) + gitrepo = CodePraise::GitRepo.new(clone_request.repo) + return if gitrepo.exists_locally? + gitrepo.clone! { |line| update_progress(clone_request.id, line) } + end + + private + + CLONE_PROGRESS = { + 'START' => 15, + 'Cloning' => 30, + 'remote' => 70, + 'Receiving' => 85, + 'Resolving' => 95, + 'Checking' => 100 + }.freeze + + def update_progress(channel_id, line) + percent = progress(line).to_s + publish(channel_id, percent) + end + + def publish(channel, message) + puts "Posting progress: #{message}" + HTTP.headers(content_type: 'application/json') + .post( + "#{CloneRepoWorker.config.API_URL}/faye", + body: { + channel: "/#{channel}", + data: message + }.to_json + ) + end + + def progress(line) + CLONE_PROGRESS[first_word_of(line)] + end + + def first_word_of(line) + line.match(/^[A-Za-z]+/).to_s end end diff --git a/workers/init.rb b/workers/init.rb index f76926d..b22d0c1 100644 --- a/workers/init.rb +++ b/workers/init.rb @@ -1,5 +1,5 @@ # frozen_string_literal: false -Dir.glob("#{File.dirname(__FILE__)}/*.rb").each do |file| +Dir.glob("#{File.dirname(__FILE__)}/*_worker.rb").each do |file| require file end diff --git a/workers/shoryuken_test.yml b/workers/shoryuken_test.yml index a7e1db1..af6f8e1 100644 --- a/workers/shoryuken_test.yml +++ b/workers/shoryuken_test.yml @@ -1,2 +1,2 @@ queues: - - https://sqs.us-east-1.amazonaws.com/503315808870/codepraise-clone_test.fifo + - https://sqs.us-east-1.amazonaws.com/503315808870/codepraise-clone_test.fifo diff --git a/workers/test_helper.rb b/workers/test_helper.rb new file mode 100644 index 0000000..d239bc6 --- /dev/null +++ b/workers/test_helper.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'webmock' +include WebMock::API + +WebMock.enable! +WebMock.allow_net_connect! + +stub_request(:post, CloneRepoWorker.config.API_URL + '/faye') + .to_return(status: 200, body: '', headers: {})