From c007d95863da1630e49e584e193b6583654893b4 Mon Sep 17 00:00:00 2001 From: Soumya Ray Date: Wed, 6 Dec 2017 14:55:08 +0800 Subject: [PATCH] Parallel processing of cloning operation - Setup SQS + Shoryuken worker - Test for new asynch processing and completion HTTP statuses - Refactor controller response representation with represent_response method - Introduce GitRepo as its own datamapper - Refactor Blame domain objects around GitRepo mapper --- Gemfile | 4 ++ Gemfile.lock | 17 +++++ Procfile | 3 +- Rakefile | 69 +++++++++++++++++-- application/controllers/app.rb | 11 +++ application/controllers/repo.rb | 17 +---- application/controllers/summary.rb | 13 +++- application/representers/repo_representer.rb | 5 +- application/services/summarize_folder.rb | 35 ++++++++++ domain/blame_reporter/blame_summary.rb | 28 -------- domain/init.rb | 3 +- .../blame_mappers}/blame_report.rb | 5 +- domain/mappers/blame_mappers/blame_summary.rb | 17 +++++ .../blame_mappers}/init.rb | 0 .../blame_mappers}/porcelain_parser.rb | 0 domain/mappers/git_mappers/git_repo.rb | 42 +++++++++++ .../git_mappers}/init.rb | 0 .../github_mappers/collaborator_mapper.rb | 0 domain/mappers/github_mappers/init.rb | 5 ++ .../github_mappers/repo_mapper.rb | 0 domain/mappers/init.rb | 7 ++ .../collaborators.rb | 0 .../for.rb | 0 .../init.rb | 1 + domain/repositories/repo_store.rb | 20 ++++++ .../repos.rb | 7 ++ .../gitrepo/{git_repo.rb => local_repo.rb} | 35 ++-------- infrastructure/gitrepo/remote_repo.rb | 34 +++++++++ init.rb | 2 +- spec/api_spec.rb | 28 +++++++- spec/spec_helper.rb | 1 + spec/summary_spec.rb | 8 ++- workers/clone_repo_worker.rb | 32 +++++++++ workers/init.rb | 5 ++ workers/load_all.rb | 6 ++ workers/shoryuken.yml | 2 + workers/shoryuken_test.yml | 2 + 37 files changed, 369 insertions(+), 95 deletions(-) create mode 100644 application/services/summarize_folder.rb delete mode 100644 domain/blame_reporter/blame_summary.rb rename domain/{blame_reporter => mappers/blame_mappers}/blame_report.rb (84%) create mode 100644 domain/mappers/blame_mappers/blame_summary.rb rename domain/{blame_reporter => mappers/blame_mappers}/init.rb (100%) rename domain/{blame_reporter => mappers/blame_mappers}/porcelain_parser.rb (100%) create mode 100644 domain/mappers/git_mappers/git_repo.rb rename domain/{github_mappers => mappers/git_mappers}/init.rb (100%) rename domain/{ => mappers}/github_mappers/collaborator_mapper.rb (100%) create mode 100644 domain/mappers/github_mappers/init.rb rename domain/{ => mappers}/github_mappers/repo_mapper.rb (100%) create mode 100644 domain/mappers/init.rb rename domain/{database_repositories => repositories}/collaborators.rb (100%) rename domain/{database_repositories => repositories}/for.rb (100%) rename domain/{database_repositories => repositories}/init.rb (78%) create mode 100644 domain/repositories/repo_store.rb rename domain/{database_repositories => repositories}/repos.rb (92%) rename infrastructure/gitrepo/{git_repo.rb => local_repo.rb} (63%) create mode 100644 infrastructure/gitrepo/remote_repo.rb create mode 100644 workers/clone_repo_worker.rb create mode 100644 workers/init.rb create mode 100644 workers/load_all.rb create mode 100644 workers/shoryuken.yml create mode 100644 workers/shoryuken_test.yml diff --git a/Gemfile b/Gemfile index f5345fd..d9d9cf5 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,10 @@ gem 'http' # Asynchronicity gems gem 'concurrent-ruby' +# Worker gems +gem 'aws-sdk-sqs', '~> 1' +gem 'shoryuken', '~> 3' + # Web app related gem 'econfig' gem 'pry' # to run console in production diff --git a/Gemfile.lock b/Gemfile.lock index 9a364ea..9688df0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,6 +4,15 @@ GEM addressable (2.5.2) public_suffix (>= 2.0.2, < 4.0) ast (2.3.0) + aws-partitions (1.44.0) + aws-sdk-core (3.11.0) + aws-partitions (~> 1.0) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-sqs (1.3.0) + aws-sdk-core (~> 3) + aws-sigv4 (~> 1.0) + aws-sigv4 (1.0.2) axiom-types (0.1.1) descendants_tracker (~> 0.0.4) ice_nine (~> 0.11.0) @@ -79,6 +88,7 @@ GEM http_parser.rb (0.6.0) ice_nine (0.11.2) inflecto (0.0.2) + jmespath (1.3.1) json (2.1.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) @@ -137,12 +147,17 @@ GEM safe_yaml (1.0.4) sequel (5.1.0) sexp_processor (4.10.0) + shoryuken (3.1.12) + aws-sdk-core (>= 2) + concurrent-ruby + thor simplecov (0.15.1) docile (~> 1.1.0) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) sqlite3 (1.3.13) + thor (0.20.0) thread_safe (0.3.6) uber (0.1.0) unf (0.1.4) @@ -165,6 +180,7 @@ PLATFORMS ruby DEPENDENCIES + aws-sdk-sqs (~> 1) concurrent-ruby database_cleaner dry-monads @@ -189,6 +205,7 @@ DEPENDENCIES roda rubocop sequel + shoryuken (~> 3) simplecov sqlite3 vcr diff --git a/Procfile b/Procfile index 9e10c0c..05817d1 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,2 @@ -web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development} \ No newline at end of file +web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development} +worker: bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken.yml \ No newline at end of file diff --git a/Rakefile b/Rakefile index fa864bc..6ad3e9c 100644 --- a/Rakefile +++ b/Rakefile @@ -10,6 +10,7 @@ end task :config do require_relative 'config/environment.rb' # load config info @app = CodePraise::Api + @config = @app.config end desc 'run tests' @@ -20,7 +21,8 @@ end desc 'rerun tests' task :respec => :config do - sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'" + puts 'REMEMBER: need to run `rake run:[dev|test]:worker` in another process' + sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'" end desc 'run application console (pry)' @@ -29,19 +31,33 @@ task :console do end namespace :run do - task :dev => :config do - sh "rerun -c 'RACK_ENV=test rackup -p 3030' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'" + namespace :api do + task :dev => :config do + puts 'REMEMBER: need to run `rake run:dev:worker` in another process' + sh "rerun -c 'rackup -p 3030' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'" + end + + task :test => :config do + puts 'REMEMBER: need to run `rake run:test:worker` in another process' + sh "rerun -c 'RACK_ENV=test rackup -p 3000' --ignore 'coverage/*' --ignore '#{@config.REPOSTORE_PATH}/*'" + end end - task :test => :config do - sh "rerun -c 'RACK_ENV=test rackup -p 3000' --ignore 'coverage/*' --ignore '#{@app.config.REPOSTORE_PATH}'" + namespace :worker do + task :dev => :config do + sh 'bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken.yml' + end + + task :test => :config do + sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/clone_repo_worker.rb -C ./workers/shoryuken_test.yml' + end end end namespace :ls do desc 'list cloned repos in repo store' task :repostore => :config do - `ls #{@app.config.REPOSTORE_PATH}` + puts `ls #{@config.REPOSTORE_PATH}` end end @@ -55,7 +71,7 @@ namespace :rm do desc 'delete cloned repos in repo store' task :repostore => :config do - sh "rm -rf #{@app.config.REPOSTORE_PATH}/*" do |ok, _| + sh "rm -rf #{@config.REPOSTORE_PATH}/*" do |ok, _| puts(ok ? 'Cloned repos deleted' : "Could not delete cloned repos") end end @@ -117,3 +133,42 @@ namespace :db do puts "Deleted #{app.config.DB_FILENAME}" end end + +namespace :queue do + require 'aws-sdk-sqs' + + desc "Create SQS queue for Shoryuken" + task :create => :config do + sqs = Aws::SQS::Client.new(region: @config.AWS_REGION) + + begin + queue = sqs.create_queue( + queue_name: @config.CLONE_QUEUE, + attributes: { + FifoQueue: 'true', + ContentBasedDeduplication: 'true' + } + ) + + q_url = sqs.get_queue_url(queue_name: @config.CLONE_QUEUE) + puts "Queue created:" + puts "Name: #{@config.CLONE_QUEUE}" + puts "Region: #{@config.AWS_REGION}" + puts "URL: #{q_url.queue_url}" + puts "Environment: #{@app.environment}" + rescue => e + puts "Error creating queue: #{e}" + end + end + + task :purge => :config do + sqs = Aws::SQS::Client.new(region: @config.AWS_REGION) + + begin + queue = sqs.purge_queue(queue_url: @config.CLONE_QUEUE_URL) + puts "Queue #{@config.CLONE_QUEUE} purged" + rescue => e + puts "Error purging queue: #{e}" + end + end +end diff --git a/application/controllers/app.rb b/application/controllers/app.rb index 4b3acc8..d21bb7d 100644 --- a/application/controllers/app.rb +++ b/application/controllers/app.rb @@ -12,6 +12,17 @@ class Api < Roda require_relative 'repo' require_relative 'summary' + def represent_response(result, representer_class) + http_response = HttpResponseRepresenter.new(result.value) + response.status = http_response.http_code + if result.success? + yield if block_given? + representer_class.new(result.value.message).to_json + else + http_response.to_json + end + end + route do |routing| response['Content-Type'] = 'application/json' diff --git a/application/controllers/repo.rb b/application/controllers/repo.rb index 60ee585..f043da7 100644 --- a/application/controllers/repo.rb +++ b/application/controllers/repo.rb @@ -34,30 +34,19 @@ class Api < Roda ownername: ownername, reponame: reponame ) - http_response = HttpResponseRepresenter.new(find_result.value) - response.status = http_response.http_code - if find_result.success? - RepoRepresenter.new(find_result.value.message).to_json - else - http_response.to_json - end + represent_response(find_result, RepoRepresenter) end # POST #{API_ROOT}/repo/:ownername/:reponame request routing.post do - service_result = LoadFromGithub.new.call( + load_result = LoadFromGithub.new.call( config: Api.config, ownername: ownername, reponame: reponame ) - http_response = HttpResponseRepresenter.new(service_result.value) - response.status = http_response.http_code - if service_result.success? + represent_response(load_result, RepoRepresenter) do response['Location'] = "#{@api_root}/repo/#{ownername}/#{reponame}" - RepoRepresenter.new(service_result.value.message).to_json - else - http_response.to_json end end end diff --git a/application/controllers/summary.rb b/application/controllers/summary.rb index f3de428..92310f2 100644 --- a/application/controllers/summary.rb +++ b/application/controllers/summary.rb @@ -17,10 +17,17 @@ class Api < Roda routing.get do path = request.remaining_path folder = path.empty? ? '' : path[1..-1] - folder_summary = Blame::Summary.new(@repo).for_folder(folder) - response.status = 200 - FolderSummaryRepresenter.new(folder_summary).to_json + request_unique = [request.env, request.path, Time.now] + request_id = (request_unique.map(&:to_s).join).hash + + summarize_result = SummarizeFolder.new.call( + repo: @repo, + folder: folder, + unique_id: request_id + ) + + represent_response(summarize_result, FolderSummaryRepresenter) end end end diff --git a/application/representers/repo_representer.rb b/application/representers/repo_representer.rb index 6fd6ad2..2e08329 100644 --- a/application/representers/repo_representer.rb +++ b/application/representers/repo_representer.rb @@ -8,9 +8,10 @@ class RepoRepresenter < Roar::Decorator include Roar::JSON property :origin_id - property :owner, extend: CollaboratorRepresenter + property :owner, extend: CollaboratorRepresenter, class: OpenStruct property :name property :git_url - collection :contributors, extend: CollaboratorRepresenter + property :size + collection :contributors, extend: CollaboratorRepresenter, class: OpenStruct end end diff --git a/application/services/summarize_folder.rb b/application/services/summarize_folder.rb new file mode 100644 index 0000000..1b53453 --- /dev/null +++ b/application/services/summarize_folder.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'dry/transaction' + +module CodePraise + # Transaction to summarize folder from local repo + class SummarizeFolder + include Dry::Transaction + + step :clone_or_find_repo + step :summarize_folder + + def clone_or_find_repo(input) + input[:gitrepo] = GitRepo.new(input[:repo]) + if input[:gitrepo].exists_locally? + Right(input) + else + repo_json = RepoRepresenter.new(input[:repo]).to_json + CloneRepoWorker.perform_async(repo_json) + Left(Result.new(:processing, 'Processing the summary request')) + end + rescue + Left(Result.new(:internal_error, 'Could not clone repo')) + end + + def summarize_folder(input) + folder_summary = Blame::Summary + .new(input[:gitrepo]) + .for_folder(input[:folder]) + Right(Result.new(:ok, folder_summary)) + rescue + Left(Result.new(:internal_error, 'Could not summarize folder')) + end + end +end \ No newline at end of file diff --git a/domain/blame_reporter/blame_summary.rb b/domain/blame_reporter/blame_summary.rb deleted file mode 100644 index 00bbe47..0000000 --- a/domain/blame_reporter/blame_summary.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module CodePraise - module Blame - # Git blame parsing and reporting services - class Summary - MAX_SIZE = 1000 # for cloning, analysis, summaries, etc. - - module Errors - TooLargeToSummarize = Class.new(StandardError) - end - - def initialize(repo) - @repo = repo - end - - def too_large? - @repo.size > MAX_SIZE - end - - def for_folder(folder_name) - raise TooLargeToSummarize if too_large? - blame_reports = Blame::Reporter.new(@repo).folder_report(folder_name) - Entity::FolderSummary.new(@repo, folder_name, blame_reports) - end - end - end -end \ No newline at end of file diff --git a/domain/init.rb b/domain/init.rb index bde80d5..91b4254 100644 --- a/domain/init.rb +++ b/domain/init.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -folders = %w[values entities - database_repositories github_mappers blame_reporter] +folders = %w[values entities mappers repositories] folders.each do |folder| require_relative "#{folder}/init.rb" diff --git a/domain/blame_reporter/blame_report.rb b/domain/mappers/blame_mappers/blame_report.rb similarity index 84% rename from domain/blame_reporter/blame_report.rb rename to domain/mappers/blame_mappers/blame_report.rb index 8b93b84..5a76333 100644 --- a/domain/blame_reporter/blame_report.rb +++ b/domain/mappers/blame_mappers/blame_report.rb @@ -6,9 +6,8 @@ module CodePraise module Blame # Git blame parsing and reporting services class Reporter - def initialize(repo, config = CodePraise::Api.config) - origin = Git::RemoteRepo.new(repo.git_url) - @local = Git::LocalRepo.new(origin, config.REPOSTORE_PATH) + def initialize(gitrepo) + @local = gitrepo.local end def folder_report(folder_name) diff --git a/domain/mappers/blame_mappers/blame_summary.rb b/domain/mappers/blame_mappers/blame_summary.rb new file mode 100644 index 0000000..b1302ce --- /dev/null +++ b/domain/mappers/blame_mappers/blame_summary.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module CodePraise + module Blame + # Git blame parsing and reporting services + class Summary + def initialize(gitrepo) + @gitrepo = gitrepo + end + + def for_folder(folder_name) + blame_reports = Blame::Reporter.new(@gitrepo).folder_report(folder_name) + Entity::FolderSummary.new(@repo, folder_name, blame_reports) + end + end + end +end \ No newline at end of file diff --git a/domain/blame_reporter/init.rb b/domain/mappers/blame_mappers/init.rb similarity index 100% rename from domain/blame_reporter/init.rb rename to domain/mappers/blame_mappers/init.rb diff --git a/domain/blame_reporter/porcelain_parser.rb b/domain/mappers/blame_mappers/porcelain_parser.rb similarity index 100% rename from domain/blame_reporter/porcelain_parser.rb rename to domain/mappers/blame_mappers/porcelain_parser.rb diff --git a/domain/mappers/git_mappers/git_repo.rb b/domain/mappers/git_mappers/git_repo.rb new file mode 100644 index 0000000..8efae36 --- /dev/null +++ b/domain/mappers/git_mappers/git_repo.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module CodePraise + class GitRepo + MAX_SIZE = 1000 # for cloning, analysis, summaries, etc. + + class Errors + NoGitRepoFound = Class.new(StandardError) + TooLargeToClone = Class.new(StandardError) + CannotOverwriteLocalRepo = Class.new(StandardError) + end + + def initialize(repo, config = CodePraise::Api.config) + @repo = repo + origin = Git::RemoteRepo.new(@repo.git_url) + @local = Git::LocalRepo.new(origin, config.REPOSTORE_PATH) + end + + def local + raise Errors::NoGitRepoFound unless exists_locally? + @local + end + + def delete! + @local.delete! + end + + def too_large? + @repo.size > MAX_SIZE + end + + def exists_locally? + @local.exists? + end + + def clone! + raise Errors::TooLargeToClone if too_large? + raise Errors::CannotOverwriteLocalRepo if exists_locally? + @local.clone_remote + end + end +end diff --git a/domain/github_mappers/init.rb b/domain/mappers/git_mappers/init.rb similarity index 100% rename from domain/github_mappers/init.rb rename to domain/mappers/git_mappers/init.rb diff --git a/domain/github_mappers/collaborator_mapper.rb b/domain/mappers/github_mappers/collaborator_mapper.rb similarity index 100% rename from domain/github_mappers/collaborator_mapper.rb rename to domain/mappers/github_mappers/collaborator_mapper.rb diff --git a/domain/mappers/github_mappers/init.rb b/domain/mappers/github_mappers/init.rb new file mode 100644 index 0000000..f76926d --- /dev/null +++ b/domain/mappers/github_mappers/init.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: false + +Dir.glob("#{File.dirname(__FILE__)}/*.rb").each do |file| + require file +end diff --git a/domain/github_mappers/repo_mapper.rb b/domain/mappers/github_mappers/repo_mapper.rb similarity index 100% rename from domain/github_mappers/repo_mapper.rb rename to domain/mappers/github_mappers/repo_mapper.rb diff --git a/domain/mappers/init.rb b/domain/mappers/init.rb new file mode 100644 index 0000000..d94a053 --- /dev/null +++ b/domain/mappers/init.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +folders = %w[github_mappers git_mappers blame_mappers] + +folders.each do |folder| + require_relative "#{folder}/init.rb" +end diff --git a/domain/database_repositories/collaborators.rb b/domain/repositories/collaborators.rb similarity index 100% rename from domain/database_repositories/collaborators.rb rename to domain/repositories/collaborators.rb diff --git a/domain/database_repositories/for.rb b/domain/repositories/for.rb similarity index 100% rename from domain/database_repositories/for.rb rename to domain/repositories/for.rb diff --git a/domain/database_repositories/init.rb b/domain/repositories/init.rb similarity index 78% rename from domain/database_repositories/init.rb rename to domain/repositories/init.rb index a99bc75..40529e7 100644 --- a/domain/database_repositories/init.rb +++ b/domain/repositories/init.rb @@ -2,4 +2,5 @@ require_relative 'repos.rb' require_relative 'collaborators.rb' +require_relative 'repo_store.rb' require_relative 'for.rb' diff --git a/domain/repositories/repo_store.rb b/domain/repositories/repo_store.rb new file mode 100644 index 0000000..ad5d9d4 --- /dev/null +++ b/domain/repositories/repo_store.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module CodePraise + module Repository + # Collection of all local git repo clones + class RepoStore + def self.all + repos = Repository::Repos.all + repos.map do |repo| + gitrepo = GitRepo.new(repo) + gitrepo.exists_locally? ? gitrepo : nil + end.compact + end + + def self.delete_all! + all.each(&:delete!) + end + end + end +end diff --git a/domain/database_repositories/repos.rb b/domain/repositories/repos.rb similarity index 92% rename from domain/database_repositories/repos.rb rename to domain/repositories/repos.rb index 750eabb..c05444c 100644 --- a/domain/database_repositories/repos.rb +++ b/domain/repositories/repos.rb @@ -36,6 +36,13 @@ def self.all Database::RepoOrm.all.map { |db_repo| rebuild_entity(db_repo) } end + def self.clone_all! + Database::RepoOrm.all.each do |repo| + gitrepo = GitRepo.new(repo) + gitrepo.clone! unless gitrepo.exists_locally? + end + end + def self.create(entity) raise 'Repo already exists' if find(entity) diff --git a/infrastructure/gitrepo/git_repo.rb b/infrastructure/gitrepo/local_repo.rb similarity index 63% rename from infrastructure/gitrepo/git_repo.rb rename to infrastructure/gitrepo/local_repo.rb index f45412a..7e54af1 100644 --- a/infrastructure/gitrepo/git_repo.rb +++ b/infrastructure/gitrepo/local_repo.rb @@ -1,42 +1,14 @@ # frozen_string_literal: true require 'fileutils' -require 'base64' +require_relative 'remote_repo.rb' module Git - # USAGE: - # load 'infrastructure/gitrepo/gitrepo.rb' - # origin = Git::RemoteRepo.new('git@github.com:soumyaray/YPBT-app.git') - # local = Git::LocalRepo.new(origin, 'infrastructure/gitrepo/repostore') module Errors # Local repo not setup or invalid InvalidLocalRepo = Class.new(StandardError) end - # Manage remote Git repository for cloning - class RemoteRepo - attr_reader :git_url - - def initialize(git_url) - @git_url = git_url - end - - def local_clone(path) - `git clone --progress #{@git_url} #{path} 2>&1` - - # Cloning into 'infrastructure/gitrepo/repostore/test_cmdline'... - # remote: Counting objects: 860, done. - # remote: Total 860 (delta 0), reused 0 (delta 0), pack-reused 860 - # Receiving objects: 100% (860/860), 543.83 KiB | 0 bytes/s, done. - # Resolving deltas: 100% (516/516), done. - # Checking connectivity... done. - end - - def unique_id - Base64.urlsafe_encode64(Digest::SHA256.digest(@git_url)) - end - end - # Manage local Git repository class LocalRepo ONLY_FOLDERS = '**/' @@ -48,7 +20,6 @@ class LocalRepo def initialize(remote, repostore_path) @remote = remote @repo_path = [repostore_path, @remote.unique_id].join('/') - clone_remote unless exists? end def clone_remote @@ -93,6 +64,10 @@ def exists? Dir.exist? @repo_path end + def delete! + FileUtils.rm_rf(@repo_path) + end + private def raise_unless_setup diff --git a/infrastructure/gitrepo/remote_repo.rb b/infrastructure/gitrepo/remote_repo.rb new file mode 100644 index 0000000..101f685 --- /dev/null +++ b/infrastructure/gitrepo/remote_repo.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'base64' + +module Git + # USAGE: + # load 'infrastructure/gitrepo/gitrepo.rb' + # origin = Git::RemoteRepo.new('git@github.com:soumyaray/YPBT-app.git') + # local = Git::LocalRepo.new(origin, 'infrastructure/gitrepo/repostore') + + # Manage remote Git repository for cloning + class RemoteRepo + attr_reader :git_url + + def initialize(git_url) + @git_url = git_url + end + + def local_clone(path) + `git clone --progress #{@git_url} #{path} 2>&1` + + # Cloning into 'infrastructure/gitrepo/repostore/test_cmdline'... + # remote: Counting objects: 860, done. + # remote: Total 860 (delta 0), reused 0 (delta 0), pack-reused 860 + # Receiving objects: 100% (860/860), 543.83 KiB | 0 bytes/s, done. + # Resolving deltas: 100% (516/516), done. + # Checking connectivity... done. + end + + def unique_id + Base64.urlsafe_encode64(Digest::SHA256.digest(@git_url)) + end + end +end diff --git a/init.rb b/init.rb index e3be2f6..39ef9d2 100644 --- a/init.rb +++ b/init.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -folders = %w[config infrastructure domain application] +folders = %w[config infrastructure domain application workers] folders.each do |folder| require_relative "#{folder}/init.rb" end diff --git a/spec/api_spec.rb b/spec/api_spec.rb index e570b6f..6b71e7f 100644 --- a/spec/api_spec.rb +++ b/spec/api_spec.rb @@ -16,7 +16,33 @@ VCR.eject_cassette end - describe 'Repo information' do + describe 'Cloning repos' do + before do + CodePraise::Repository::RepoStore.delete_all! + CodePraise::LoadFromGithub.new.call( + config: app.config, + ownername: USERNAME, + reponame: REPO_NAME + ) + end + + after do + CodePraise::Repository::Repos.clone_all! + end + + it 'HAPPY: it should begin processing uncloned repo' do + get "#{API_VER}/summary/#{USERNAME}/#{REPO_NAME}" + _(last_response.status).must_equal 202 + + STDOUT.sync + 5.times { sleep(1); print '.' } + + get "#{API_VER}/summary/#{USERNAME}/#{REPO_NAME}" + _(last_response.status).must_equal 200 + end + end + + describe 'Cloned repo information' do before do app.DB[:repos_contributors].delete app.DB[:repos].delete diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index db58a13..4985904 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -24,6 +24,7 @@ VCR.configure do |c| c.cassette_library_dir = CASSETTES_FOLDER c.hook_into :webmock + c.ignore_hosts 'sqs.us-east-1.amazonaws.com' github_token = app.config.GH_TOKEN c.filter_sensitive_data('') { github_token } diff --git a/spec/summary_spec.rb b/spec/summary_spec.rb index 81341bb..6cdf0e3 100644 --- a/spec/summary_spec.rb +++ b/spec/summary_spec.rb @@ -20,7 +20,9 @@ reponame: REPO_NAME ) - @repo = CodePraise::Repository::Repos.find_full_name(USERNAME, REPO_NAME) + repo = CodePraise::Repository::Repos.find_full_name(USERNAME, REPO_NAME) + @gitrepo = CodePraise::GitRepo.new(repo) + @gitrepo.clone! unless @gitrepo.exists_locally? end after do @@ -28,7 +30,7 @@ end it 'HAPPY: should get blame summary for entire repo' do - summary = CodePraise::Blame::Summary.new(@repo).for_folder('') + summary = CodePraise::Blame::Summary.new(@gitrepo).for_folder('') _(summary.subfolders.count).must_equal 10 _(summary.base_files.count).must_equal 1 _(summary.base_files.keys.first).must_equal 'init.rb' @@ -36,7 +38,7 @@ end it 'HAPPY: should get accurate blame summary for specific folder' do - summary = CodePraise::Blame::Summary.new(@repo).for_folder('forms') + summary = CodePraise::Blame::Summary.new(@gitrepo).for_folder('forms') _(summary.subfolders.count).must_equal 2 _(summary.subfolders['errors']['']).must_equal({name: "luyimin", count: 2}) diff --git a/workers/clone_repo_worker.rb b/workers/clone_repo_worker.rb new file mode 100644 index 0000000..38f812b --- /dev/null +++ b/workers/clone_repo_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative 'load_all' + +require 'econfig' +require 'shoryuken' + +# Shoryuken worker class to clone repos in parallel +class CloneRepoWorker + extend Econfig::Shortcut + Econfig.env = ENV['RACK_ENV'] || 'development' + Econfig.root = File.expand_path('..', File.dirname(__FILE__)) + + Shoryuken.sqs_client = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + + include Shoryuken::Worker + shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true + + def perform(_sqs_msg, worker_request) + request = CodePraise::RepoRepresenter.new(OpenStruct.new).from_json worker_request + puts "REQUEST: #{request}" + gitrepo = CodePraise::GitRepo.new(request) + puts "EXISTS: #{gitrepo.exists_locally?}" + gitrepo.clone! + puts "REQUEST: #{request}" + puts "EXISTS: #{gitrepo.exists_locally?}" + end +end diff --git a/workers/init.rb b/workers/init.rb new file mode 100644 index 0000000..f76926d --- /dev/null +++ b/workers/init.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: false + +Dir.glob("#{File.dirname(__FILE__)}/*.rb").each do |file| + require file +end diff --git a/workers/load_all.rb b/workers/load_all.rb new file mode 100644 index 0000000..1ef8bf4 --- /dev/null +++ b/workers/load_all.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +folders = %w[config infrastructure domain application] +folders.each do |folder| + require_relative "../#{folder}/init.rb" +end diff --git a/workers/shoryuken.yml b/workers/shoryuken.yml new file mode 100644 index 0000000..02d6e6a --- /dev/null +++ b/workers/shoryuken.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.us-east-1.amazonaws.com/503315808870/codepraise-clone.fifo diff --git a/workers/shoryuken_test.yml b/workers/shoryuken_test.yml new file mode 100644 index 0000000..a7e1db1 --- /dev/null +++ b/workers/shoryuken_test.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.us-east-1.amazonaws.com/503315808870/codepraise-clone_test.fifo