Skip to content

Commit

Permalink
Merge pull request #505 from datacite/event-facets
Browse files Browse the repository at this point in the history 
Improve performance of events api
  • Loading branch information
Martin Fenner authored May 7, 2020
2 parents e14fc8f + 64f0eea commit 91387a2
Show file tree
Hide file tree
Showing 13 changed files with 103 additions and 246 deletions.
2 changes: 1 addition & 1 deletion app/controllers/clients_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def totals
page = { size: 0, number: 1 }
state = current_user.present? && current_user.is_admin_or_staff? && params[:state].present? ? params[:state] : "registered,findable"
response = Doi.query(nil, provider_id: params[:provider_id], state: state, page: page, totals_agg: "client")
registrant = response.results.total.positive? ? clients_totals(response.response.aggregations.clients_totals.buckets) : []
registrant = response.results.total.positive? ? clients_totals(response.aggregations.clients_totals.buckets) : []

render json: registrant, status: :ok
end
Expand Down
157 changes: 29 additions & 128 deletions app/controllers/concerns/facetable.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,18 +121,6 @@ def facet_by_resource_type(arr)
end
end

def facet_by_year_month(arr)
arr.map do |hsh|
month = hsh["key_as_string"][5..6].to_i
title = I18n.t("date.month_names")[month] + " " + hsh["key_as_string"][0..3]

{ "id" => hsh["key_as_string"][0..6],
"title" => title,
"count" => hsh["doc_count"],
"sum" => hsh.dig("total_by_year_month", "value") }
end
end

def facet_by_source(arr)
arr.map do |hsh|
{ "id" => hsh["key"],
Expand All @@ -141,170 +129,83 @@ def facet_by_source(arr)
end
end

def facet_citations_by_year(hash)
arr = hash.dig('years', 'buckets').map do |h|
year = h['key']
title = h['key'].to_i.to_s

{
'id' => year,
'title' => title,
'sum' => h.dig('total_by_year', 'value') }
end
{ "count" => hash.dig("sum_distribution", "value"),
"years" => arr }
end

def facet_citations_by_year_v1(hash)
arr = hash.dig('years', 'buckets').map do |h|
year = h["key_as_string"][0..3].to_i
title = h["key_as_string"][0..3]

{
'id' => year,
'title' => title,
'sum' => h.dig('total_by_year', 'value') }
end
{ "count" => hash.dig("sum_distribution", "value"),
"years" => arr }
end

def facet_counts_by_year_month(hash)
arr = hash.dig('year_months', 'buckets').map do |h|
month = h["key_as_string"][5..6].to_i
title = I18n.t("date.month_names")[month] + " " + h["key_as_string"][0..3]

{
"id" => h["key_as_string"][0..6],
'title' => title,
'sum' => h.dig('total_by_year_month', 'value') }
end
{ "count" => hash.dig("sum_distribution", "value"),
"yearMonths" => arr }
end

def facet_by_relation_type(arr)
arr.map do |hsh|
arr = hsh.dig("year_months", "buckets").map do |h|
month = h["key_as_string"][5..6].to_i
title = I18n.t("date.month_names")[month] + " " + h["key_as_string"][0..3]

year_month_arr = hsh.dig("year_month", "buckets").map do |h|
{
"id" => h["key_as_string"][0..6],
"title" => title,
"sum" => h.dig("total_by_year_month", "value") }
"id" => h["key_as_string"],
"title" => h["key_as_string"],
"sum" => h["doc_count"] }
end

{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh.dig("sum_distribution", "value"),
"yearMonths" => arr }
"count" => hsh["doc_count"],
"yearMonths" => year_month_arr }
end
end

def facet_by_relation_type_v1(arr)
arr.map do |hsh|
arr = hsh.dig("year_months", "buckets").map do |h|
month = h["key_as_string"][5..6].to_i
title = I18n.t("date.month_names")[month] + " " + h["key_as_string"][0..3]

year_month_arr = hsh.dig("year_month", "buckets").map do |h|
{
"id" => h["key_as_string"][0..6],
"title" => title,
"sum" => h.dig("total_by_year_month", "value") }
"id" => h["key_as_string"],
"title" => h["key_as_string"],
"sum" => h["doc_count"] }
end

{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh.dig("sum_distribution", "value"),
"year-months" => arr }
"count" => hsh["doc_count"],
"year-months" => year_month_arr }
end
end

def facet_by_citation_type(arr)
arr.map do |hsh|
arr = hsh.dig("year_months", "buckets").map do |h|
month = h["key_as_string"][5..6].to_i
title = I18n.t("date.month_names")[month] + " " + h["key_as_string"][0..3]

year_month_arr = hsh.dig("year_month", "buckets").map do |h|
{
"id" => h["key_as_string"][0..6],
"title" => title,
"sum" => h.dig("total_by_year_month", "value") }
"id" => h["key_as_string"],
"title" => h["key_as_string"],
"sum" => h["doc_count"] }
end

{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh["doc_count"],
"year-months" => arr }
"yearMonths" => year_month_arr }
end
end

def facet_by_pairings(arr)
def facet_by_citation_type_v1(arr)
arr.map do |hsh|
arr = hsh.dig("recipient", "buckets").map do |h|
title = h["key"]
year_month_arr = hsh.dig("year_month", "buckets").map do |h|
{
"id" => h["key"],
"title" => title,
"sum" => h.dig("total", "value") }
"id" => h["key_as_string"],
"title" => h["key_as_string"],
"sum" => h["doc_count"] }
end
arr.reject! {|h| h["id"] == hsh["key"]}

{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh["doc_count"],
"registrants" => arr }
"year-months" => year_month_arr }
end
end

def facet_by_registrants(arr)
arr.map do |hsh|
arr = hsh.dig("year", "buckets").map do |h|
title = h["key_as_string"][0..3]

year_arr = hsh.dig("year", "buckets").map do |h|
{
"id" => h["key_as_string"][0..3],
"title" => title,
"sum" => h.dig("total_by_year", "value") }
"id" => h["key_as_string"],
"title" => h["key_as_string"],
"sum" => h["doc_count"] }
end

{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh["doc_count"],
"years" => arr }
end
end

def facet_by_metric_type(arr)
arr.map do |hsh|
{ "id" => hsh["key"],
"title" => hsh["key"].gsub(/-/, ' ').titleize,
"count" => hsh["doc_count"] }
end
end

def facet_by_dois(arr)
arr.map do |hsh|
arr = hsh.dig("relation_types", "buckets").map do |h|
title = h["key"]
{
"id" => h["key"],
"title" => title,
"sum" => h.dig("total_by_type", "value") }
end
arr.reject! {|h| h["id"] == hsh["key"]}
{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh["doc_count"],
"relationTypes" => arr }
end
end

def facet_citations_by_dois(arr)
arr.map do |hsh|
{ "id" => hsh["key"],
"title" => hsh["key"],
"count" => hsh.dig("total", "value")}
"years" => year_arr }
end
end

Expand Down
18 changes: 6 additions & 12 deletions app/controllers/events_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,16 +139,12 @@ def index

render json: EventSerializer.new(results, options).serialized_json, status: :ok
else
aggregations = params.fetch(:aggregations, "") || ""

sources = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.sources.buckets) : nil
prefixes = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil
citation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil
relation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil
registrants = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil
pairings = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_pairings(response.response.aggregations.pairings.buckets) : nil
states = total.positive? && aggregations.include?("state_aggregations") ? facet_by_source(response.response.aggregations.states.buckets) : nil

sources = total.positive? ? facet_by_source(response.response.aggregations.sources.buckets) : nil
prefixes = total.positive? ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil
citation_types = total.positive? ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil
relation_types = total.positive? ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil
registrants = total.positive? ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil

results = response.results

options = {}
Expand All @@ -160,9 +156,7 @@ def index
prefixes: prefixes,
"citationTypes" => citation_types,
"relationTypes" => relation_types,
pairings: pairings,
registrants: registrants,
"states": states,
}.compact

options[:links] = {
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/export_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ def repositories
response = Doi.query(nil, state: "registered,findable", page: { size: 0, number: 1 }, totals_agg: "client_export")

client_totals = {}
totals_buckets = response.response.aggregations.clients_totals.buckets
totals_buckets = response.aggregations.clients_totals.buckets
totals_buckets.each do |totals|
client_totals[totals["key"]] = {
"count" => totals["doc_count"],
Expand Down
18 changes: 2 additions & 16 deletions app/controllers/old_events_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ class OldEventsController < ApplicationController
prepend_before_action :authenticate_user!, except: [:index, :show]
before_action :detect_crawler
before_action :load_event, only: [:show, :destroy]
before_action :set_include, only: [:index, :show, :create, :update]
authorize_resource only: [:destroy]

def create
Expand Down Expand Up @@ -56,7 +55,6 @@ def update

def show
options = {}
options[:include] = @include
options[:is_collection] = false

render json: OldEventSerializer.new(@event, options).serialized_json, status: :ok
Expand Down Expand Up @@ -132,10 +130,9 @@ def index
else
sources = total > 0 ? facet_by_source(response.aggregations.sources.buckets) : nil
prefixes = total > 0 ? facet_by_source(response.aggregations.prefixes.buckets) : nil
citation_types = total > 0 ? facet_by_citation_type(response.aggregations.citation_types.buckets) : nil
citation_types = total > 0 ? facet_by_citation_type_v1(response.aggregations.citation_types.buckets) : nil
relation_types = total > 0 ? facet_by_relation_type_v1(response.aggregations.relation_types.buckets) : nil
registrants = total > 0 && params[:extra] ? facet_by_registrants(response.aggregations.registrants.buckets) : nil
pairings = total > 0 && params[:extra] ? facet_by_pairings(response.aggregations.pairings.buckets) : nil
registrants = total > 0 ? facet_by_registrants(response.aggregations.registrants.buckets) : nil

results = response.results

Expand All @@ -148,7 +145,6 @@ def index
prefixes: prefixes,
"citation-types" => citation_types,
"relation-types" => relation_types,
pairings: pairings,
registrants: registrants
}.compact

Expand All @@ -173,7 +169,6 @@ def index
"page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil,
"page[size]" => page[:size] }.compact.to_query
}.compact
options[:include] = @include
options[:is_collection] = true

render json: OldEventSerializer.new(results, options).serialized_json, status: :ok
Expand All @@ -197,15 +192,6 @@ def load_event
fail ActiveRecord::RecordNotFound unless @event.present?
end

def set_include
if params[:include].present?
@include = params[:include].split(",").map { |i| i.downcase.underscore.to_sym }
@include = @include & [:subj, :obj]
else
@include = []
end
end

private

def safe_params
Expand Down
14 changes: 7 additions & 7 deletions app/controllers/repositories_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ def index
begin
total = response.results.total
total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0
years = total > 0 ? facet_by_year(response.response.aggregations.years.buckets) : nil
providers = total > 0 ? facet_by_combined_key(response.response.aggregations.providers.buckets) : nil
software = total > 0 ? facet_by_software(response.response.aggregations.software.buckets) : nil
certificates = total > 0 ? facet_by_key(response.response.aggregations.certificates.buckets) : nil
client_types = total > 0 ? facet_by_key(response.response.aggregations.client_types.buckets) : nil
repository_types = total > 0 ? facet_by_key(response.response.aggregations.repository_types.buckets) : nil
years = total > 0 ? facet_by_year(response.aggregations.years.buckets) : nil
providers = total > 0 ? facet_by_combined_key(response.aggregations.providers.buckets) : nil
software = total > 0 ? facet_by_software(response.aggregations.software.buckets) : nil
certificates = total > 0 ? facet_by_key(response.aggregations.certificates.buckets) : nil
client_types = total > 0 ? facet_by_key(response.aggregations.client_types.buckets) : nil
repository_types = total > 0 ? facet_by_key(response.aggregations.repository_types.buckets) : nil

respond_to do |format|
format.json do
Expand Down Expand Up @@ -194,7 +194,7 @@ def totals

state = current_user.present? && current_user.is_admin_or_staff? && params[:state].present? ? params[:state] : "registered,findable"
response = Doi.query(nil, provider_id: params[:provider_id], state: state, page: page, totals_agg: "client")
registrant = response.results.total.positive? ? clients_totals(response.response.aggregations.clients_totals.buckets) : []
registrant = response.results.total.positive? ? clients_totals(response.aggregations.clients_totals.buckets) : []

render json: registrant, status: :ok
end
Expand Down
Loading

0 comments on commit 91387a2

Please sign in to comment.