Skip to content

Commit

Permalink
scroll api support for activities and events. #371
Browse files Browse the repository at this point in the history
  • Loading branch information
Martin Fenner committed Dec 8, 2019
1 parent a933dae commit cb2178c
Show file tree
Hide file tree
Showing 2 changed files with 156 additions and 114 deletions.
71 changes: 46 additions & 25 deletions app/controllers/activities_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,36 +19,57 @@ def index
elsif params[:ids].present?
response = Activity.find_by_id(params[:ids], page: page, sort: sort)
else
response = Activity.query(params[:query], uid: params[:doi_id], page: page, sort: sort)
response = Activity.query(params[:query], uid: params[:doi_id], page: page, sort: sort, scroll_id: params[:scroll_id])
end

begin
total = response.results.total
total_for_pages = page[:cursor].nil? ? total.to_f : [total.to_f, 10000].min
total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0

@activities = response.results

options = {}
options[:meta] = {
total: total,
"totalPages" => total_pages,
page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil,
}.compact

options[:links] = {
self: request.original_url,
next: @activities.size < page[:size] ? nil : request.base_url + "/activities?" + {
query: params[:query],
"page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(@activities.to_a.last[:sort]).join(","), padding: false) : nil,
"page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil,
"page[size]" => page[:size],
sort: params[:sort] }.compact.to_query
if page[:scroll].present?
results = response.results
total = response.total
else
total = response.results.total
total_for_pages = page[:cursor].nil? ? total.to_f : [total.to_f, 10000].min
total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0
end

if page[:scroll].present?
options = {}
options[:meta] = {
total: total,
"scroll-id" => response.scroll_id,
}.compact
options[:links] = {
self: request.original_url,
next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/activities?" + {
"scroll-id" => response.scroll_id,
"page[scroll]" => "3m",
"page[size]" => page[:size] }.compact.to_query
}.compact
options[:is_collection] = true

render json: ActivitySerializer.new(results, options).serialized_json, status: :ok
else
options = {}
options[:meta] = {
total: total,
"totalPages" => total_pages,
page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil,
}.compact
options[:include] = @include
options[:is_collection] = true

render json: ActivitySerializer.new(@activities, options).serialized_json, status: :ok
options[:links] = {
self: request.original_url,
next: response.results.size < page[:size] ? nil : request.base_url + "/activities?" + {
query: params[:query],
"page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(@activities.to_a.last[:sort]).join(","), padding: false) : nil,
"page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil,
"page[size]" => page[:size],
sort: params[:sort] }.compact.to_query
}.compact
options[:include] = @include
options[:is_collection] = true

render json: ActivitySerializer.new(response.results, options).serialized_json, status: :ok
end
rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception
Raven.capture_exception(exception)

Expand Down
199 changes: 110 additions & 89 deletions app/controllers/events_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ class EventsController < ApplicationController
include BatchLoaderHelper
require 'benchmark'



prepend_before_action :authenticate_user!, except: [:index, :show]
before_action :load_event, only: [:show, :destroy]
before_action :set_include, only: [:index, :show, :create, :update]
Expand Down Expand Up @@ -104,102 +102,125 @@ def index
year_month: params[:year_month],
aggregations: params[:aggregations],
unique: params[:unique],
scroll_id: params[:scroll_id],
page: page,
sort: sort)
end

total = response.results.total
total_for_pages = page[:cursor].nil? ? [total.to_f, 10000].min : total.to_f
total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0


aggregations = params.fetch(:aggregations, "") || ""

sources = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.sources.buckets) : nil
prefixes = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil
citation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil
relation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil
registrants = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil
pairings = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_pairings(response.response.aggregations.pairings.buckets) : nil
dois = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois.buckets) : nil
dois_usage = total.positive? ? EventsQuery.new.usage(params[:doi]) : nil
# dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil
citations = total.positive? ? EventsQuery.new.citations(params[:doi]) : nil
citations_histogram = total.positive? ? EventsQuery.new.citations_histogram(params[:doi]) : nil
references = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil
relations = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil

views_histogram = total.positive? ? EventsQuery.new.views_histogram(params[:doi]) : nil
downloads_histogram = total.positive? ? EventsQuery.new.downloads_histogram(params[:doi]) : nil

# views = total.positive? ? EventsQuery.new.views(params[:doi]) : nil
# downloads = total.positive? ? EventsQuery.new.downloads(params[:doi]) : nil
unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil
unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil

results = response.results
if page[:scroll].present?
results = response.results
total = response.total
else
total = response.results.total
total_for_pages = page[:cursor].nil? ? [total.to_f, 10000].min : total.to_f
total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0
end

options = {}
options[:meta] = {
total: total,
"totalPages" => total_pages,
page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil,
sources: sources,
prefixes: prefixes,
"citationTypes" => citation_types,
"relationTypes" => relation_types,
pairings: pairings,
registrants: registrants,
"doisRelationTypes": dois,
"doisUsageTypes": dois_usage,
# "doisCitations": dois_citations,
"citationsHistogram": citations_histogram,
"uniqueCitations": citations,
"references": references,
"relations": relations,
"uniqueNodes": {
"objCount": unique_obj_count,
"subjCount": unique_subj_count
},
"viewsHistogram": views_histogram,
# "views": views,
"downloadsHistogram": downloads_histogram
# "downloads": downloads
}.compact

options[:links] = {
self: request.original_url,
next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + {
"query" => params[:query],
"subj-id" => params[:subj_id],
"obj-id" => params[:obj_id],
"doi" => params[:doi],
"orcid" => params[:orcid],
"prefix" => params[:prefix],
"subtype" => params[:subtype],
"citation_type" => params[:citation_type],
"source-id" => params[:source_id],
"relation-type-id" => params[:relation_type_id],
"issn" => params[:issn],
"registrant-id" => params[:registrant_id],
"publication-year" => params[:publication_year],
"year-month" => params[:year_month],
"page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(results.to_a.last[:sort]).join(","), padding: false) : nil,
"page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil,
"page[size]" => page[:size] }.compact.to_query
if page[:scroll].present?
options = {}
options[:meta] = {
total: total,
"scroll-id" => response.scroll_id,
}.compact
options[:links] = {
self: request.original_url,
next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + {
"scroll-id" => response.scroll_id,
"page[scroll]" => "3m",
"page[size]" => page[:size] }.compact.to_query
}.compact
options[:is_collection] = true

render json: EventSerializer.new(results, options).serialized_json, status: :ok
else
aggregations = params.fetch(:aggregations, "") || ""

sources = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.sources.buckets) : nil
prefixes = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil
citation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil
relation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil
registrants = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil
pairings = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_pairings(response.response.aggregations.pairings.buckets) : nil
dois = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois.buckets) : nil
dois_usage = total.positive? ? EventsQuery.new.usage(params[:doi]) : nil
# dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil
citations = total.positive? ? EventsQuery.new.citations(params[:doi]) : nil
citations_histogram = total.positive? ? EventsQuery.new.citations_histogram(params[:doi]) : nil
references = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil
relations = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil

views_histogram = total.positive? ? EventsQuery.new.views_histogram(params[:doi]) : nil
downloads_histogram = total.positive? ? EventsQuery.new.downloads_histogram(params[:doi]) : nil

# views = total.positive? ? EventsQuery.new.views(params[:doi]) : nil
# downloads = total.positive? ? EventsQuery.new.downloads(params[:doi]) : nil
unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil
unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil

results = response.results

options[:is_collection] = true

events_serialized = EventSerializer.new(results, options).serializable_hash
options = {}
options[:meta] = {
total: total,
"totalPages" => total_pages,
page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil,
sources: sources,
prefixes: prefixes,
"citationTypes" => citation_types,
"relationTypes" => relation_types,
pairings: pairings,
registrants: registrants,
"doisRelationTypes": dois,
"doisUsageTypes": dois_usage,
# "doisCitations": dois_citations,
"citationsHistogram": citations_histogram,
"uniqueCitations": citations,
"references": references,
"relations": relations,
"uniqueNodes": {
"objCount": unique_obj_count,
"subjCount": unique_subj_count
},
"viewsHistogram": views_histogram,
# "views": views,
"downloadsHistogram": downloads_histogram
# "downloads": downloads
}.compact

if @include.include?(:dois)
options[:include] = []
doi_names = (results.map { |event| event.doi}).uniq().join(",")
events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), {is_collection: true}).serializable_hash.dig(:data)
options[:links] = {
self: request.original_url,
next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + {
"query" => params[:query],
"subj-id" => params[:subj_id],
"obj-id" => params[:obj_id],
"doi" => params[:doi],
"orcid" => params[:orcid],
"prefix" => params[:prefix],
"subtype" => params[:subtype],
"citation_type" => params[:citation_type],
"source-id" => params[:source_id],
"relation-type-id" => params[:relation_type_id],
"issn" => params[:issn],
"registrant-id" => params[:registrant_id],
"publication-year" => params[:publication_year],
"year-month" => params[:year_month],
"page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(results.to_a.last[:sort]).join(","), padding: false) : nil,
"page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil,
"page[size]" => page[:size] }.compact.to_query
}.compact

options[:is_collection] = true

events_serialized = EventSerializer.new(results, options).serializable_hash

if @include.include?(:dois)
options[:include] = []
doi_names = (results.map { |event| event.doi}).uniq().join(",")
events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), {is_collection: true}).serializable_hash.dig(:data)
end

render json: events_serialized, status: :ok
end

render json: events_serialized, status: :ok
end

def destroy
Expand Down

0 comments on commit cb2178c

Please sign in to comment.