From 573d4b0b787537e38ba20e92b64dd57f8c4104e7 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Tue, 9 Apr 2019 00:49:42 +0200 Subject: [PATCH] optimize performance of facets. #224 --- app/controllers/activities_controller.rb | 2 +- app/controllers/clients_controller.rb | 2 +- app/controllers/concerns/facetable.rb | 73 +++++++++++----------- app/controllers/data_centers_controller.rb | 2 +- app/controllers/dois_controller.rb | 2 +- app/controllers/members_controller.rb | 2 +- app/controllers/providers_controller.rb | 2 +- app/controllers/works_controller.rb | 4 +- 8 files changed, 43 insertions(+), 46 deletions(-) diff --git a/app/controllers/activities_controller.rb b/app/controllers/activities_controller.rb index ea280bd9d..7e72255bf 100644 --- a/app/controllers/activities_controller.rb +++ b/app/controllers/activities_controller.rb @@ -27,7 +27,7 @@ def index total_for_pages = page[:cursor].present? ? total.to_f : [total.to_f, 10000].min total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 - @activities = response.results.results + @activities = response.results options = {} options[:meta] = { diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index be38da997..8e031772c 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -35,7 +35,7 @@ def index providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil software = total > 0 ? facet_by_software(response.response.aggregations.software.buckets) : nil - @clients = response.results.results + @clients = response.results options = {} options[:meta] = { diff --git a/app/controllers/concerns/facetable.rb b/app/controllers/concerns/facetable.rb index 157de7d10..44aa921e3 100644 --- a/app/controllers/concerns/facetable.rb +++ b/app/controllers/concerns/facetable.rb @@ -88,14 +88,11 @@ def facet_by_provider(arr) # generate hash with id and name for each provider in facet ids = arr.map { |hsh| hsh["key"] }.join(",") - providers = Provider.find_by_ids(ids, size: 1000).results.reduce({}) do |sum, p| - sum[p.symbol.downcase] = p.name - sum - end + providers = Provider.find_by_ids(ids, size: 1000).records.pluck(:symbol, :name).to_h arr.map do |hsh| { "id" => hsh["key"], - "title" => providers[hsh["key"]], + "title" => providers[hsh["key"].upcase], "count" => hsh["doc_count"] } end end @@ -104,14 +101,11 @@ def providers_totals(arr) # generate hash with id and name for each provider in facet ids = arr.map { |hsh| hsh["key"] }.join(",") - providers = Provider.find_by_ids(ids, size: 1000).results.reduce({}) do |sum, p| - sum[p.symbol.downcase] = p.name - sum - end + providers = Provider.find_by_ids(ids, size: 1000).records.pluck(:symbol, :name).to_h arr.map do |hsh| { "id" => hsh["key"], - "title" => providers[hsh["key"]], + "title" => providers[hsh["key"].upcase], "count" => hsh["doc_count"], "temporal" => { "this_month" => facet_anual(hsh.this_month.buckets), @@ -140,38 +134,44 @@ def prefixes_totals(arr) def clients_totals(arr) - # generate hash with id and name for each provider in facet - - ids = arr.map { |hsh| hsh["key"] }.join(",") - clients = Client.find_by_ids(ids, size: 2000).results.reduce({}) do |sum, p| - sum[p.symbol.downcase] = p.name - sum - end + logger = Logger.new(STDOUT) - arr.map do |hsh| - { "id" => hsh["key"], - "title" => clients[hsh["key"]], - "count" => hsh["doc_count"], - "temporal" => { - "this_month" => facet_anual(hsh.this_month.buckets), - "this_year" => facet_anual(hsh.this_year.buckets), - "last_year" => facet_anual(hsh.last_year.buckets)}, - "states" => facet_by_key(hsh.states.buckets) - } - end + # generate hash with id and name for each client in facet + ids = nil + logger.info "[Benchmark] clients_totals ids " + Benchmark.ms { + ids = arr.map { |hsh| hsh["key"] }.join(",") + }.to_s + " ms" + + clients = nil + logger.info "[Benchmark] clients_totals find_by_ids " + Benchmark.ms { + clients = Client.find_by_ids(ids, size: 2000).records.pluck(:symbol, :name).to_h + }.to_s + " ms" + + logger.info "[Benchmark] clients_totals map " + Benchmark.ms { + arr = arr.map do |hsh| + { "id" => hsh["key"], + "title" => clients[hsh["key"].upcase], + "count" => hsh["doc_count"], + "temporal" => { + "this_month" => facet_anual(hsh.this_month.buckets), + "this_year" => facet_anual(hsh.this_year.buckets), + "last_year" => facet_anual(hsh.last_year.buckets) + }, + "states" => facet_by_key(hsh.states.buckets) + } + end + }.to_s + " ms" + arr end def facet_by_provider_ids(arr) # generate hash with id and name for each provider in facet ids = arr.map { |hsh| hsh["key"] }.join(",") - providers = Provider.find_by_id_list(ids).results.reduce({}) do |sum, p| - sum[p.id] = p.name - sum - end + providers = Provider.find_by_id_list(ids).records.pluck(:symbol, :name).to_h arr.map do |hsh| { "id" => hsh["key"], - "title" => providers[hsh["key"]], + "title" => providers[hsh["key"].upcase], "count" => hsh["doc_count"] } end end @@ -179,14 +179,11 @@ def facet_by_provider_ids(arr) def facet_by_client(arr) # generate hash with id and name for each client in facet ids = arr.map { |hsh| hsh["key"] }.join(",") - clients = Client.find_by_ids(ids).results.reduce({}) do |sum, p| - sum[p.symbol.downcase] = p.name - sum - end + clients = Client.find_by_ids(ids).records.pluck(:symbol, :name).to_h arr.map do |hsh| { "id" => hsh["key"], - "title" => clients[hsh["key"]], + "title" => clients[hsh["key"].upcase], "count" => hsh["doc_count"] } end end diff --git a/app/controllers/data_centers_controller.rb b/app/controllers/data_centers_controller.rb index 55e4ca548..83908e902 100644 --- a/app/controllers/data_centers_controller.rb +++ b/app/controllers/data_centers_controller.rb @@ -28,7 +28,7 @@ def index years = total > 0 ? facet_by_year(response.response.aggregations.years.buckets) : nil providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil - @clients = response.results.results + @clients = response.results options = {} options[:meta] = { diff --git a/app/controllers/dois_controller.rb b/app/controllers/dois_controller.rb index 51106247d..27cd79298 100644 --- a/app/controllers/dois_controller.rb +++ b/app/controllers/dois_controller.rb @@ -94,7 +94,7 @@ def index else results = nil logger.info "[Benchmark] results " + Benchmark.ms { - results = response.results.results + results = response.results }.to_s + " ms" total = response.results.total total_for_pages = page[:cursor].present? ? total.to_f : [total.to_f, 10000].min diff --git a/app/controllers/members_controller.rb b/app/controllers/members_controller.rb index 94c1e50e5..bd96144e6 100644 --- a/app/controllers/members_controller.rb +++ b/app/controllers/members_controller.rb @@ -29,7 +29,7 @@ def index organization_types = total > 0 ? facet_by_key(response.response.aggregations.organization_types.buckets) : nil focus_areas = total > 0 ? facet_by_key(response.response.aggregations.focus_areas.buckets) : nil - @members = response.results.results + @members = response.results options = {} options[:meta] = { diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index 4dbd2b00a..d5c17f2e8 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -35,7 +35,7 @@ def index organization_types = total > 0 ? facet_by_key(response.response.aggregations.organization_types.buckets) : nil focus_areas = total > 0 ? facet_by_key(response.response.aggregations.focus_areas.buckets) : nil - @providers = response.results.results + @providers = response.results options = {} options[:meta] = { diff --git a/app/controllers/works_controller.rb b/app/controllers/works_controller.rb index ff8a5c9a1..284ab1bb8 100644 --- a/app/controllers/works_controller.rb +++ b/app/controllers/works_controller.rb @@ -58,7 +58,7 @@ def index providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil clients = total > 0 ? facet_by_client(response.response.aggregations.clients.buckets) : nil - @dois = response.results.results + @dois = response.results options = {} options[:meta] = { @@ -100,7 +100,7 @@ def index if sample_dois @dois = sample_dois else - @dois = response.results.results + @dois = response.results end render json: WorkSerializer.new(@dois, options).serialized_json, status: :ok rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception