diff --git a/Gemfile b/Gemfile index 9b468c02f..608438dcc 100644 --- a/Gemfile +++ b/Gemfile @@ -59,6 +59,7 @@ gem 'git', '~> 1.5' group :development, :test do gem 'rspec-rails', '~> 3.5', '>= 3.5.2' + gem 'rspec-benchmark', '~> 0.4.0' gem "better_errors" gem "binding_of_caller" gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] diff --git a/Gemfile.lock b/Gemfile.lock index bba2c2658..a919b206f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,25 +3,25 @@ GEM specs: aasm (5.0.2) concurrent-ruby (~> 1.0) - actioncable (5.2.2) - actionpack (= 5.2.2) + actioncable (5.2.3) + actionpack (= 5.2.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.2) - actionpack (= 5.2.2) - actionview (= 5.2.2) - activejob (= 5.2.2) + actionmailer (5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.2) - actionview (= 5.2.2) - activesupport (= 5.2.2) + actionpack (5.2.3) + actionview (= 5.2.3) + activesupport (= 5.2.3) rack (~> 2.0) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.2) - activesupport (= 5.2.2) + actionview (5.2.3) + activesupport (= 5.2.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -31,25 +31,25 @@ GEM activemodel (>= 4.1, < 6) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) - activejob (5.2.2) - activesupport (= 5.2.2) + activejob (5.2.3) + activesupport (= 5.2.3) globalid (>= 0.3.6) - activemodel (5.2.2) - activesupport (= 5.2.2) - activerecord (5.2.2) - activemodel (= 5.2.2) - activesupport (= 5.2.2) + activemodel (5.2.3) + activesupport (= 5.2.3) + activerecord (5.2.3) + activemodel (= 5.2.3) + activesupport (= 5.2.3) arel (>= 9.0) - activestorage (5.2.2) - actionpack (= 5.2.2) - activerecord (= 5.2.2) + activestorage (5.2.3) + actionpack (= 5.2.3) + activerecord (= 5.2.3) marcel (~> 0.3.1) - activesupport (5.2.2) + activesupport (5.2.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - addressable (2.5.2) + addressable (2.6.0) public_suffix (>= 2.0.2, < 4.0) ansi (1.5.0) api-pagination (4.8.2) @@ -57,25 +57,28 @@ GEM audited (4.8.0) activerecord (>= 4.0, < 5.3) aws-eventstream (1.0.2) - aws-partitions (1.144.0) - aws-sdk-core (3.46.2) - aws-eventstream (~> 1.0) + aws-partitions (1.149.0) + aws-sdk-core (3.48.3) + aws-eventstream (~> 1.0, >= 1.0.2) aws-partitions (~> 1.0) - aws-sigv4 (~> 1.0) + aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-kms (1.13.0) - aws-sdk-core (~> 3, >= 3.39.0) - aws-sigv4 (~> 1.0) - aws-sdk-s3 (1.31.0) - aws-sdk-core (~> 3, >= 3.39.0) + aws-sdk-kms (1.16.0) + aws-sdk-core (~> 3, >= 3.48.2) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.36.0) + aws-sdk-core (~> 3, >= 3.48.2) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.0) - aws-sdk-sqs (1.10.0) - aws-sdk-core (~> 3, >= 3.39.0) - aws-sigv4 (~> 1.0) - aws-sigv4 (1.0.3) + aws-sdk-sqs (1.13.0) + aws-sdk-core (~> 3, >= 3.48.2) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.1.0) + aws-eventstream (~> 1.0, >= 1.0.2) base32-url (0.5) bcrypt (3.1.12) + benchmark-perf (0.4.0) + benchmark-trend (0.2.0) benchmark_methods (0.7) bergamasco (0.3.17) activesupport (>= 4.2.5, < 6) @@ -118,12 +121,12 @@ GEM rdf-rdfxml (~> 2.0) rdf-turtle (~> 2.2) thor (~> 0.19) - bootsnap (1.4.1) + bootsnap (1.4.2) msgpack (~> 1.0) builder (3.2.3) - byebug (11.0.0) + byebug (11.0.1) cancancan (2.3.0) - capybara (3.14.0) + capybara (3.16.1) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -162,7 +165,7 @@ GEM csl (~> 1.0) css_parser (1.7.0) addressable - dalli (2.7.9) + dalli (2.7.10) database_cleaner (1.7.0) debug_inspector (0.0.3) diff-lcs (1.3) @@ -171,16 +174,16 @@ GEM docopt (0.6.1) domain_name (0.5.20180417) unf (>= 0.0.5, < 1.0.0) - dotenv (2.7.1) + dotenv (2.7.2) ebnf (1.1.3) rdf (~> 3.0) sxp (~> 1.0) edtf (3.0.4) activesupport (>= 3.0, < 6.0) - elasticsearch (6.1.0) - elasticsearch-api (= 6.1.0) - elasticsearch-transport (= 6.1.0) - elasticsearch-api (6.1.0) + elasticsearch (6.3.0) + elasticsearch-api (= 6.3.0) + elasticsearch-transport (= 6.3.0) + elasticsearch-api (6.3.0) multi_json elasticsearch-extensions (0.0.31) ansi @@ -190,7 +193,7 @@ GEM elasticsearch (> 1) hashie elasticsearch-rails (6.0.0) - elasticsearch-transport (6.1.0) + elasticsearch-transport (6.3.0) faraday multi_json equivalent-xml (0.6.0) @@ -299,7 +302,7 @@ GEM method_source (0.9.2) mime-types (3.2.2) mime-types-data (~> 3.2015) - mime-types-data (3.2018.0812) + mime-types-data (3.2019.0331) mimemagic (0.3.3) mini_magick (4.9.3) mini_mime (1.0.1) @@ -307,7 +310,7 @@ GEM minitest (5.11.3) money (6.13.2) i18n (>= 0.6.4, <= 2) - msgpack (1.2.8) + msgpack (1.2.9) multi_json (1.13.1) multipart-post (2.0.0) mysql2 (0.4.10) @@ -319,9 +322,9 @@ GEM oj (2.18.5) oj_mimic_json (1.0.1) pandoc-ruby (2.0.2) - postrank-uri (1.0.23) - addressable (>= 2.3.0, < 2.6) - nokogiri (>= 1.6.1, < 1.9) + postrank-uri (1.0.24) + addressable (>= 2.4.0) + nokogiri (>= 1.8.0) public_suffix (>= 2.0.0, < 2.1) premailer (1.11.1) addressable @@ -331,35 +334,35 @@ GEM pwqgen.rb (0.1.0) docopt (~> 0.5) sysrandom - rack (2.0.6) + rack (2.0.7) rack-accept (0.4.5) rack (>= 0.4) - rack-cors (1.0.2) + rack-cors (1.0.3) rack-test (1.1.0) rack (>= 1.0, < 3) rack-utf8_sanitizer (1.6.0) rack (>= 1.0, < 3.0) - rails (5.2.2) - actioncable (= 5.2.2) - actionmailer (= 5.2.2) - actionpack (= 5.2.2) - actionview (= 5.2.2) - activejob (= 5.2.2) - activemodel (= 5.2.2) - activerecord (= 5.2.2) - activestorage (= 5.2.2) - activesupport (= 5.2.2) + rails (5.2.3) + actioncable (= 5.2.3) + actionmailer (= 5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) + activemodel (= 5.2.3) + activerecord (= 5.2.3) + activestorage (= 5.2.3) + activesupport (= 5.2.3) bundler (>= 1.3.0) - railties (= 5.2.2) + railties (= 5.2.3) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.0.4) loofah (~> 2.2, >= 2.2.2) - railties (5.2.2) - actionpack (= 5.2.2) - activesupport (= 5.2.2) + railties (5.2.3) + actionpack (= 5.2.3) + activesupport (= 5.2.3) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) @@ -388,13 +391,21 @@ GEM rdf (>= 2.2, < 4.0) rdf-xsd (3.0.1) rdf (~> 3.0) - regexp_parser (1.3.0) + regexp_parser (1.4.0) request_store (1.4.1) rack (>= 1.4) rest-client (2.0.2) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-benchmark (0.4.0) + benchmark-perf (~> 0.4.0) + benchmark-trend (~> 0.2.0) + rspec (>= 3.0.0, < 4.0.0) rspec-core (3.8.0) rspec-support (~> 3.8.0) rspec-expectations (3.8.2) @@ -544,6 +555,7 @@ DEPENDENCIES rack-utf8_sanitizer (~> 1.6) rails (~> 5.2.0) rake (~> 12.0) + rspec-benchmark (~> 0.4.0) rspec-rails (~> 3.5, >= 3.5.2) sentry-raven (~> 2.9) shoryuken (~> 4.0) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index 831a47af2..5be3063c8 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -128,13 +128,22 @@ def destroy end def totals + logger = Logger.new(STDOUT) + page = { size: 0, number: 1} - response = Doi.query(nil, provider_id: params[:provider_id], page: page, totals_agg: true) + response = nil + logger.info "[Benchmark] clients totals " + Benchmark.ms { + response = Doi.query(nil, provider_id: params[:provider_id], page: page, totals_agg: true) + }.to_s + " ms" total = response.results.total - registrant = total > 0 ? clients_totals(response.response.aggregations.clients_totals.buckets) : nil - - render json: registrant, status: :ok + registrant = nil + logger.info "[Benchmark] clients clients_totals " + Benchmark.ms { + registrant = total > 0 ? clients_totals(response.response.aggregations.clients_totals.buckets) : nil + }.to_s + " ms" + logger.info "[Benchmark] clients render " + Benchmark.ms { + render json: registrant, status: :ok + }.to_s + " ms" end protected diff --git a/app/controllers/concerns/facetable.rb b/app/controllers/concerns/facetable.rb index befe5f2b6..157de7d10 100644 --- a/app/controllers/concerns/facetable.rb +++ b/app/controllers/concerns/facetable.rb @@ -144,7 +144,6 @@ def clients_totals(arr) ids = arr.map { |hsh| hsh["key"] }.join(",") clients = Client.find_by_ids(ids, size: 2000).results.reduce({}) do |sum, p| - puts sum sum[p.symbol.downcase] = p.name sum end diff --git a/app/controllers/dois_controller.rb b/app/controllers/dois_controller.rb index 93d230552..33f517b5a 100644 --- a/app/controllers/dois_controller.rb +++ b/app/controllers/dois_controller.rb @@ -1,5 +1,6 @@ require 'uri' require 'base64' +require 'benchmark' class DoisController < ApplicationController include ActionController::MimeResponds @@ -13,6 +14,8 @@ class DoisController < ApplicationController def index authorize! :read, Doi + logger = Logger.new(STDOUT) + sort = case params[:sort] when "name" then { "doi" => { order: 'asc' }} when "-name" then { "doi" => { order: 'desc' }} @@ -35,34 +38,38 @@ def index end if params[:id].present? - response = Doi.find_by_id(params[:id]) + logger.info "[Benchmark] find_by_id " + Benchmark.ms { + response = Doi.find_by_id(params[:id]) + }.to_s + " ms" elsif params[:ids].present? response = Doi.find_by_ids(params[:ids], page: page, sort: sort) else - response = Doi.query(params[:query], - state: params[:state], - created: params[:created], - registered: params[:registered], - provider_id: params[:provider_id], - client_id: params[:client_id], - prefix: params[:prefix], - person_id: params[:person_id], - resource_type_id: params[:resource_type_id], - schema_version: params[:schema_version], - subject: params[:subject], - link_check_status: params[:link_check_status], - link_check_has_schema_org: params[:link_check_has_schema_org], - link_check_body_has_pid: params[:link_check_body_has_pid], - link_check_found_schema_org_id: params[:link_check_found_schema_org_id], - link_check_found_dc_identifier: params[:link_check_found_dc_identifier], - link_check_found_citation_doi: params[:link_check_found_citation_doi], - link_check_redirect_count_gte: params[:link_check_redirect_count_gte], - sample_group: sample_group_field, - sample_size: params[:sample], - source: params[:source], - page: page, - sort: sort, - random: params[:random]) + logger.info "[Benchmark] query " + Benchmark.ms { + response = Doi.query(params[:query], + state: params[:state], + created: params[:created], + registered: params[:registered], + provider_id: params[:provider_id], + client_id: params[:client_id], + prefix: params[:prefix], + person_id: params[:person_id], + resource_type_id: params[:resource_type_id], + schema_version: params[:schema_version], + subject: params[:subject], + link_check_status: params[:link_check_status], + link_check_has_schema_org: params[:link_check_has_schema_org], + link_check_body_has_pid: params[:link_check_body_has_pid], + link_check_found_schema_org_id: params[:link_check_found_schema_org_id], + link_check_found_dc_identifier: params[:link_check_found_dc_identifier], + link_check_found_citation_doi: params[:link_check_found_citation_doi], + link_check_redirect_count_gte: params[:link_check_redirect_count_gte], + sample_group: sample_group_field, + sample_size: params[:sample], + source: params[:source], + page: page, + sort: sort, + random: params[:random]) + }.to_s + " ms" end begin @@ -83,28 +90,48 @@ def index total = sample_dois.length total_pages = 1 else - results = response.results.results + logger.info "[Benchmark] results " + Benchmark.ms { + results = response.results.results + }.to_s + " ms" total = response.results.total total_for_pages = page[:cursor].present? ? total.to_f : [total.to_f, 10000].min total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 end - states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil - resource_types = total > 0 ? facet_by_resource_type(response.response.aggregations.resource_types.buckets) : nil - created = total > 0 ? facet_by_year(response.response.aggregations.created.buckets) : nil - registered = total > 0 ? facet_by_year(response.response.aggregations.registered.buckets) : nil - providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil - clients = total > 0 ? facet_by_client(response.response.aggregations.clients.buckets) : nil - prefixes = total > 0 ? facet_by_key(response.response.aggregations.prefixes.buckets) : nil - schema_versions = total > 0 ? facet_by_schema(response.response.aggregations.schema_versions.buckets) : nil - sources = total > 0 ? facet_by_key(response.response.aggregations.sources.buckets) : nil - link_checks_status = total > 0 ? facet_by_cumulative_year(response.response.aggregations.link_checks_status.buckets) : nil - links_with_schema_org = total > 0 ? facet_by_cumulative_year(response.response.aggregations.link_checks_has_schema_org.buckets) : nil - link_checks_schema_org_id = total > 0 ? response.response.aggregations.link_checks_schema_org_id.value : nil - link_checks_dc_identifier = total > 0 ? response.response.aggregations.link_checks_dc_identifier.value : nil - link_checks_citation_doi = total > 0 ? response.response.aggregations.link_checks_citation_doi.value : nil - links_checked = total > 0 ? response.response.aggregations.links_checked.value : nil - subjects = total > 0 ? facet_by_key(response.response.aggregations.subjects.buckets) : nil + states = nil + resource_types = nil + created = nil + registered = nil + providers = nil + clients = nil + prefixes = nil + schema_versions = nil + sources = nil + link_checks_status = nil + links_with_schema_org = nil + link_checks_schema_org_id = nil + link_checks_dc_identifier = nil + link_checks_citation_doi = nil + links_checked = nil + subjects = nil + logger.info "[Benchmark] aggregations " + Benchmark.ms { + states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil + resource_types = total > 0 ? facet_by_resource_type(response.response.aggregations.resource_types.buckets) : nil + created = total > 0 ? facet_by_year(response.response.aggregations.created.buckets) : nil + registered = total > 0 ? facet_by_year(response.response.aggregations.registered.buckets) : nil + providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil + clients = total > 0 ? facet_by_client(response.response.aggregations.clients.buckets) : nil + prefixes = total > 0 ? facet_by_key(response.response.aggregations.prefixes.buckets) : nil + schema_versions = total > 0 ? facet_by_schema(response.response.aggregations.schema_versions.buckets) : nil + sources = total > 0 ? facet_by_key(response.response.aggregations.sources.buckets) : nil + link_checks_status = total > 0 ? facet_by_cumulative_year(response.response.aggregations.link_checks_status.buckets) : nil + links_with_schema_org = total > 0 ? facet_by_cumulative_year(response.response.aggregations.link_checks_has_schema_org.buckets) : nil + link_checks_schema_org_id = total > 0 ? response.response.aggregations.link_checks_schema_org_id.value : nil + link_checks_dc_identifier = total > 0 ? response.response.aggregations.link_checks_dc_identifier.value : nil + link_checks_citation_doi = total > 0 ? response.response.aggregations.link_checks_citation_doi.value : nil + links_checked = total > 0 ? response.response.aggregations.links_checked.value : nil + subjects = total > 0 ? facet_by_key(response.response.aggregations.subjects.buckets) : nil + }.to_s + " ms" respond_to do |format| format.json do @@ -148,7 +175,9 @@ def index :current_ability => current_ability, } - render json: DoiSerializer.new(@dois, options).serialized_json, status: :ok + logger.info "[Benchmark] render " + Benchmark.ms { + render json: DoiSerializer.new(@dois, options).serialized_json, status: :ok + }.to_s + " ms" end format.citation do # fetch formatted citations @@ -215,7 +244,6 @@ def create # logger.info safe_params.inspect @doi = Doi.new(safe_params) - puts @doi.inspect # capture username and password for reuse in the handle system @doi.current_user = current_user diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index c8d5bfa95..6d65314cd 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -121,15 +121,22 @@ def update end def totals - page = { size: 0, number: 1} - - response = Doi.query("", page: page, totals_agg: true) + logger = Logger.new(STDOUT) + page = { size: 0, number: 1} + response = nil + logger.info "[Benchmark] providers totals " + Benchmark.ms { + response = Doi.query("", page: page, totals_agg: true) + }.to_s + " ms" total = response.results.total - registrant = total > 0 ? providers_totals(response.response.aggregations.providers_totals.buckets) : nil - - render json: registrant, status: :ok + registrant = nil + logger.info "[Benchmark] providers providers_totals " + Benchmark.ms { + registrant = total > 0 ? providers_totals(response.response.aggregations.providers_totals.buckets) : nil + }.to_s + " ms" + logger.info "[Benchmark] providers render " + Benchmark.ms { + render json: registrant, status: :ok + }.to_s + " ms" end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 8029923b0..898a882f3 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -17,6 +17,7 @@ require "database_cleaner" require 'aasm/rspec' require "strip_attributes/matchers" +require 'rspec-benchmark' # Checks for pending migration and applies them before tests are run. ActiveRecord::Migration.maintain_test_schema! @@ -37,6 +38,7 @@ RSpec.configure do |config| config.include FactoryBot::Syntax::Methods config.include StripAttributes::Matchers + config.include RSpec::Benchmark::Matchers # don't use transactions, use database_clear gem via support file config.use_transactional_fixtures = false diff --git a/spec/requests/dois_spec.rb b/spec/requests/dois_spec.rb index 75722a4b7..2866e3ea7 100644 --- a/spec/requests/dois_spec.rb +++ b/spec/requests/dois_spec.rb @@ -697,7 +697,6 @@ it 'revert the changes with status code 200' do post "/dois/undo", params: undo_attributes.to_json, headers: headers - puts response.body expect(response).to have_http_status(200) end end @@ -820,7 +819,6 @@ before { put "/dois/#{doi.doi}", params: valid_attributes.to_json, headers: admin_headers } it 'returns no errors' do - puts response.body expect(response).to have_http_status(200) expect(json.dig('data', 'attributes', 'doi')).to eq(doi.doi) end @@ -1024,7 +1022,6 @@ end it 'returns status code 201' do - puts response.body expect(response).to have_http_status(201) end @@ -1061,7 +1058,6 @@ end it 'returns status code 201' do - puts response.body expect(response).to have_http_status(201) end