From cb2178cd87a8d866d23b166b25fc920a1e00d703 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Sun, 8 Dec 2019 17:28:06 +0000 Subject: [PATCH] scroll api support for activities and events. #371 --- app/controllers/activities_controller.rb | 71 +++++--- app/controllers/events_controller.rb | 199 +++++++++++++---------- 2 files changed, 156 insertions(+), 114 deletions(-) diff --git a/app/controllers/activities_controller.rb b/app/controllers/activities_controller.rb index 1e5fe18c4..c1d542208 100644 --- a/app/controllers/activities_controller.rb +++ b/app/controllers/activities_controller.rb @@ -19,36 +19,57 @@ def index elsif params[:ids].present? response = Activity.find_by_id(params[:ids], page: page, sort: sort) else - response = Activity.query(params[:query], uid: params[:doi_id], page: page, sort: sort) + response = Activity.query(params[:query], uid: params[:doi_id], page: page, sort: sort, scroll_id: params[:scroll_id]) end begin - total = response.results.total - total_for_pages = page[:cursor].nil? ? total.to_f : [total.to_f, 10000].min - total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 - - @activities = response.results - - options = {} - options[:meta] = { - total: total, - "totalPages" => total_pages, - page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil, - }.compact - - options[:links] = { - self: request.original_url, - next: @activities.size < page[:size] ? nil : request.base_url + "/activities?" + { - query: params[:query], - "page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(@activities.to_a.last[:sort]).join(","), padding: false) : nil, - "page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil, - "page[size]" => page[:size], - sort: params[:sort] }.compact.to_query + if page[:scroll].present? + results = response.results + total = response.total + else + total = response.results.total + total_for_pages = page[:cursor].nil? ? total.to_f : [total.to_f, 10000].min + total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 + end + + if page[:scroll].present? + options = {} + options[:meta] = { + total: total, + "scroll-id" => response.scroll_id, + }.compact + options[:links] = { + self: request.original_url, + next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/activities?" + { + "scroll-id" => response.scroll_id, + "page[scroll]" => "3m", + "page[size]" => page[:size] }.compact.to_query + }.compact + options[:is_collection] = true + + render json: ActivitySerializer.new(results, options).serialized_json, status: :ok + else + options = {} + options[:meta] = { + total: total, + "totalPages" => total_pages, + page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil, }.compact - options[:include] = @include - options[:is_collection] = true - render json: ActivitySerializer.new(@activities, options).serialized_json, status: :ok + options[:links] = { + self: request.original_url, + next: response.results.size < page[:size] ? nil : request.base_url + "/activities?" + { + query: params[:query], + "page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(@activities.to_a.last[:sort]).join(","), padding: false) : nil, + "page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil, + "page[size]" => page[:size], + sort: params[:sort] }.compact.to_query + }.compact + options[:include] = @include + options[:is_collection] = true + + render json: ActivitySerializer.new(response.results, options).serialized_json, status: :ok + end rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception Raven.capture_exception(exception) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index cf4e915e5..f16f71fe0 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -6,8 +6,6 @@ class EventsController < ApplicationController include BatchLoaderHelper require 'benchmark' - - prepend_before_action :authenticate_user!, except: [:index, :show] before_action :load_event, only: [:show, :destroy] before_action :set_include, only: [:index, :show, :create, :update] @@ -104,102 +102,125 @@ def index year_month: params[:year_month], aggregations: params[:aggregations], unique: params[:unique], + scroll_id: params[:scroll_id], page: page, sort: sort) end - total = response.results.total - total_for_pages = page[:cursor].nil? ? [total.to_f, 10000].min : total.to_f - total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 - - - aggregations = params.fetch(:aggregations, "") || "" - - sources = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.sources.buckets) : nil - prefixes = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil - citation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil - relation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil - registrants = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil - pairings = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_pairings(response.response.aggregations.pairings.buckets) : nil - dois = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois.buckets) : nil - dois_usage = total.positive? ? EventsQuery.new.usage(params[:doi]) : nil - # dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil - citations = total.positive? ? EventsQuery.new.citations(params[:doi]) : nil - citations_histogram = total.positive? ? EventsQuery.new.citations_histogram(params[:doi]) : nil - references = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil - relations = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil - - views_histogram = total.positive? ? EventsQuery.new.views_histogram(params[:doi]) : nil - downloads_histogram = total.positive? ? EventsQuery.new.downloads_histogram(params[:doi]) : nil - - # views = total.positive? ? EventsQuery.new.views(params[:doi]) : nil - # downloads = total.positive? ? EventsQuery.new.downloads(params[:doi]) : nil - unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil - unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil - - results = response.results + if page[:scroll].present? + results = response.results + total = response.total + else + total = response.results.total + total_for_pages = page[:cursor].nil? ? [total.to_f, 10000].min : total.to_f + total_pages = page[:size] > 0 ? (total_for_pages / page[:size]).ceil : 0 + end - options = {} - options[:meta] = { - total: total, - "totalPages" => total_pages, - page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil, - sources: sources, - prefixes: prefixes, - "citationTypes" => citation_types, - "relationTypes" => relation_types, - pairings: pairings, - registrants: registrants, - "doisRelationTypes": dois, - "doisUsageTypes": dois_usage, - # "doisCitations": dois_citations, - "citationsHistogram": citations_histogram, - "uniqueCitations": citations, - "references": references, - "relations": relations, - "uniqueNodes": { - "objCount": unique_obj_count, - "subjCount": unique_subj_count - }, - "viewsHistogram": views_histogram, - # "views": views, - "downloadsHistogram": downloads_histogram - # "downloads": downloads - }.compact - - options[:links] = { - self: request.original_url, - next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + { - "query" => params[:query], - "subj-id" => params[:subj_id], - "obj-id" => params[:obj_id], - "doi" => params[:doi], - "orcid" => params[:orcid], - "prefix" => params[:prefix], - "subtype" => params[:subtype], - "citation_type" => params[:citation_type], - "source-id" => params[:source_id], - "relation-type-id" => params[:relation_type_id], - "issn" => params[:issn], - "registrant-id" => params[:registrant_id], - "publication-year" => params[:publication_year], - "year-month" => params[:year_month], - "page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(results.to_a.last[:sort]).join(","), padding: false) : nil, - "page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil, - "page[size]" => page[:size] }.compact.to_query + if page[:scroll].present? + options = {} + options[:meta] = { + total: total, + "scroll-id" => response.scroll_id, }.compact + options[:links] = { + self: request.original_url, + next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + { + "scroll-id" => response.scroll_id, + "page[scroll]" => "3m", + "page[size]" => page[:size] }.compact.to_query + }.compact + options[:is_collection] = true + + render json: EventSerializer.new(results, options).serialized_json, status: :ok + else + aggregations = params.fetch(:aggregations, "") || "" + + sources = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.sources.buckets) : nil + prefixes = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_source(response.response.aggregations.prefixes.buckets) : nil + citation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_citation_type(response.response.aggregations.citation_types.buckets) : nil + relation_types = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_relation_type(response.response.aggregations.relation_types.buckets) : nil + registrants = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_registrants(response.response.aggregations.registrants.buckets) : nil + pairings = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_pairings(response.response.aggregations.pairings.buckets) : nil + dois = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois.buckets) : nil + dois_usage = total.positive? ? EventsQuery.new.usage(params[:doi]) : nil + # dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil + citations = total.positive? ? EventsQuery.new.citations(params[:doi]) : nil + citations_histogram = total.positive? ? EventsQuery.new.citations_histogram(params[:doi]) : nil + references = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil + relations = total.positive? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil + + views_histogram = total.positive? ? EventsQuery.new.views_histogram(params[:doi]) : nil + downloads_histogram = total.positive? ? EventsQuery.new.downloads_histogram(params[:doi]) : nil + + # views = total.positive? ? EventsQuery.new.views(params[:doi]) : nil + # downloads = total.positive? ? EventsQuery.new.downloads(params[:doi]) : nil + unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil + unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil + + results = response.results - options[:is_collection] = true - - events_serialized = EventSerializer.new(results, options).serializable_hash + options = {} + options[:meta] = { + total: total, + "totalPages" => total_pages, + page: page[:cursor].nil? && page[:number].present? ? page[:number] : nil, + sources: sources, + prefixes: prefixes, + "citationTypes" => citation_types, + "relationTypes" => relation_types, + pairings: pairings, + registrants: registrants, + "doisRelationTypes": dois, + "doisUsageTypes": dois_usage, + # "doisCitations": dois_citations, + "citationsHistogram": citations_histogram, + "uniqueCitations": citations, + "references": references, + "relations": relations, + "uniqueNodes": { + "objCount": unique_obj_count, + "subjCount": unique_subj_count + }, + "viewsHistogram": views_histogram, + # "views": views, + "downloadsHistogram": downloads_histogram + # "downloads": downloads + }.compact - if @include.include?(:dois) - options[:include] = [] - doi_names = (results.map { |event| event.doi}).uniq().join(",") - events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), {is_collection: true}).serializable_hash.dig(:data) + options[:links] = { + self: request.original_url, + next: results.size < page[:size] || page[:size] == 0 ? nil : request.base_url + "/events?" + { + "query" => params[:query], + "subj-id" => params[:subj_id], + "obj-id" => params[:obj_id], + "doi" => params[:doi], + "orcid" => params[:orcid], + "prefix" => params[:prefix], + "subtype" => params[:subtype], + "citation_type" => params[:citation_type], + "source-id" => params[:source_id], + "relation-type-id" => params[:relation_type_id], + "issn" => params[:issn], + "registrant-id" => params[:registrant_id], + "publication-year" => params[:publication_year], + "year-month" => params[:year_month], + "page[cursor]" => page[:cursor] ? Base64.urlsafe_encode64(Array.wrap(results.to_a.last[:sort]).join(","), padding: false) : nil, + "page[number]" => page[:cursor].nil? && page[:number].present? ? page[:number] + 1 : nil, + "page[size]" => page[:size] }.compact.to_query + }.compact + + options[:is_collection] = true + + events_serialized = EventSerializer.new(results, options).serializable_hash + + if @include.include?(:dois) + options[:include] = [] + doi_names = (results.map { |event| event.doi}).uniq().join(",") + events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), {is_collection: true}).serializable_hash.dig(:data) + end + + render json: events_serialized, status: :ok end - - render json: events_serialized, status: :ok end def destroy