From a3a0071598dfbd1bbb764c389493b56a1d96678b Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 25 Nov 2019 18:44:33 +0100 Subject: [PATCH 01/10] add queyr object --- app/models/event.rb | 15 +++++++++++++ app/queries/events_query.rb | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 app/queries/events_query.rb diff --git a/app/models/event.rb b/app/models/event.rb index cab438262..27e917919 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -312,6 +312,21 @@ def self.citations_aggregations(doi) } end + + def self.citation_count_aggregation(doi) + # doi = Event.new.normalize_doi(doi) if doi.present? + # citations_filter = { script: { script: "(#{PASSIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['subj_id'].value) || (#{ACTIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['obj_id'].value)" } } + # references_filter = { script: { script: "(#{PASSIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['obj_id'].value) || (#{ACTIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['subj_id'].value)" } } + + { + citations: { + terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} + } + } + + end + + def self.advanced_aggregations(doi=nil) { unique_obj_count: { cardinality: { field: 'obj_id' }}, diff --git a/app/queries/events_query.rb b/app/queries/events_query.rb new file mode 100644 index 000000000..f5e576be6 --- /dev/null +++ b/app/queries/events_query.rb @@ -0,0 +1,45 @@ +class EventsQuery + attr_reader :relation + + include Facetable + + ACTIVE_RELATION_TYPES = [ + "cites", + "is-supplement-to", + "references" + ] + + PASSIVE_RELATION_TYPES = [ + "is-cited-by", + "is-supplemented-by", + "is-referenced-by" + ] + + + + def initialize(relation = Event.new) + @relation = relation + end + + def doi_citations(doi) + pid = relation.normalize_doi(doi) + + query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" + + results = Event.query(query, aggregations: "citation_count_aggregation", page: { size: 1, cursor: [] }).response.aggregations.citations.buckets + + + results.any? ? results.first.total.value : 0 + end + + def citations(doi) + doi.downcase.split(",").map do |item| + {id: item, total: EventsQuery.new.doi_citations(item)} + end + end + + + + # private + +end \ No newline at end of file From a6fced02ec6af8270c7593d16ea8817028e11b3f Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 25 Nov 2019 18:44:44 +0100 Subject: [PATCH 02/10] interfaces_change --- app/controllers/events_controller.rb | 4 +++- app/graphql/types/metric_interface.rb | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 36c6d76ff..b8f4ea316 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -85,6 +85,7 @@ def index elsif params[:ids].present? response = Event.find_by_id(params[:ids], page: page, sort: sort) else + # params[:query] = '(subj_id:"https://doi.org/10.11646/zootaxa.4263.2.11" AND (relation_type_id:cites OR relation_type_id:is-supplement-to OR relation_type_id:references)) OR (obj_id:"https://doi.org/10.11646/zootaxa.4263.2.11" AND (relation_type_id:is-cited-by OR relation_type_id:is-supplemented-by OR relation_type_id:is-referenced-by))' response = Event.query(params[:query], subj_id: params[:subj_id], obj_id: params[:obj_id], @@ -123,7 +124,8 @@ def index dois_usage = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois_usage.dois.buckets) : nil dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil citations_histogram = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_year(response.response.aggregations.citations_histogram) : nil - citations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.citations.dois.buckets) : nil + citations = EventsQuery.new.citations(params[:doi]) + # citations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.citations.dois.buckets) : nil references = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil relations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil views_histogram = total.positive? && aggregations.include?("metrics_aggregations") ? facet_counts_by_year_month(response.response.aggregations.views_histogram) : nil diff --git a/app/graphql/types/metric_interface.rb b/app/graphql/types/metric_interface.rb index 523a35ef4..097acd871 100644 --- a/app/graphql/types/metric_interface.rb +++ b/app/graphql/types/metric_interface.rb @@ -29,8 +29,7 @@ def download_count end def citation_count - meta = citations_aggs - meta.first.fetch("total", {}).fetch("value", nil) if meta.any? + EventsQuery.new.doi_citations(doi_from_url(object.identifier)) end def reference_count From 1fd742258f120a2a77b82d63812df88dea084c8a Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 25 Nov 2019 18:44:53 +0100 Subject: [PATCH 03/10] specs --- spec/queries/events_query_spec.rb | 21 ++++++++++++ spec/requests/events_spec.rb | 55 +++++++++++++++++-------------- 2 files changed, 51 insertions(+), 25 deletions(-) create mode 100644 spec/queries/events_query_spec.rb diff --git a/spec/queries/events_query_spec.rb b/spec/queries/events_query_spec.rb new file mode 100644 index 000000000..7a535c22b --- /dev/null +++ b/spec/queries/events_query_spec.rb @@ -0,0 +1,21 @@ +require 'rails_helper' + +describe EventsQuery, elasticsearch: true do + # before(:each) { allow(Time.zone).to receive(:now).and_return(Time.mktime(2015, 4, 8)) } + + context "citation events" do + subject { create(:event_for_datacite_related) } + + it "doi_citations" do + expect(EventsQuery.new.doi_citations(subject.obj_id.gsub("https://doi.org/",""))).to eq(1) + end + + it "doi_citations wiht 0 citations" do + expect(EventsQuery.new.doi_citations("10.5061/dryad.dd47sd5/1")).to eq(0) + end + + it "citations" do + expect(EventsQuery.new.citations("10.5061/dryad.47sd5/1,10.5061/dryad.47sd5/2,10.5061/dryad.47sd5/3")).to eq(1) + end + end +end diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index af1f0a41d..fe1136e62 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -699,35 +699,40 @@ end end - # # We cannot do this anymore as the aggregation needs to be done per doi - # context "unique citations for a list of dois" do - # let!(:event) { create_list(:event_for_datacite_related, 50) } - # let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } - # let(:dois) { ((event.map{ |e| e['subj_id'].gsub("https://doi.org/","")})[1, 20]).join(",")} - # let(:uri) { "/events?aggregations=metrics_aggregations&dois=#{dois}" } - - # before do - # Event.import - # sleep 1 - # end + # We cannot do this anymore as the aggregation needs to be done per doi + context "unique citations for a list of dois" do + let!(:event) { create_list(:event_for_datacite_related, 50, relation_type_id: "is-cited-by") } + let(:doi) { "https://doi.org/10.5061/dryad.47sd5/1".gsub("https://doi.org/","")} + let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } + let(:dois) { ((event.map{ |e| e['subj_id'].gsub("https://doi.org/","")})[1, 20]).join(",")} + let(:uri) { "/events?aggregations=metrics_aggregations&doi=#{dois}" } - # # Exclude the token header. - # let(:headers) do - # { "HTTP_ACCEPT" => "application/vnd.api+json; version=2" } - # end + before do + Event.import + sleep 1 + end - # it "json" do - # get uri, nil, headers + # Exclude the token header. + let(:headers) do + { "HTTP_ACCEPT" => "application/vnd.api+json; version=2" } + end - # expect(last_response.status).to eq(200) - # response = JSON.parse(last_response.body) - # citations = (response.dig("meta", "uniqueCitations")).select { |item| item["id"] == doi } - # total = response.dig("meta", "total") + it "json" do + get uri, nil, headers - # expect(total).to eq(51) - # expect((citations.select { |doi| dois.split(",").include?(doi["id"]) }).length).to eq(20) - # end - # end + puts event[2].inspect + + + expect(last_response.status).to eq(200) + response = JSON.parse(last_response.body) + puts response.dig("meta", "uniqueCitations") + citations = (response.dig("meta", "uniqueCitations")).select { |item| item["id"] == doi } + total = response.dig("meta", "total") + + expect(total).to eq(51) + expect((citations.select { |doi| dois.split(",").include?(doi["id"]) }).length).to eq(20) + end + end # Just test that the API can be accessed without a token. # context "with no API key" do From fecbdaac0d404bb9caf7b6d00cc9b66cf39ffccb Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 26 Nov 2019 16:03:02 +0100 Subject: [PATCH 04/10] specs --- app/controllers/events_controller.rb | 2 +- app/queries/events_query.rb | 12 ++------ spec/queries/events_query_spec.rb | 20 ++++++++++--- spec/requests/events_spec.rb | 44 +++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index b8f4ea316..1439df7f1 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -124,7 +124,7 @@ def index dois_usage = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_by_dois(response.response.aggregations.dois_usage.dois.buckets) : nil dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil citations_histogram = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_year(response.response.aggregations.citations_histogram) : nil - citations = EventsQuery.new.citations(params[:doi]) + citations = params[:doi].present? ? EventsQuery.new.citations(params[:doi]) : [] # citations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.citations.dois.buckets) : nil references = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil relations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil diff --git a/app/queries/events_query.rb b/app/queries/events_query.rb index f5e576be6..ca24fd39e 100644 --- a/app/queries/events_query.rb +++ b/app/queries/events_query.rb @@ -15,26 +15,20 @@ class EventsQuery "is-referenced-by" ] - - def initialize(relation = Event.new) @relation = relation end def doi_citations(doi) pid = relation.normalize_doi(doi) - - query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" - - results = Event.query(query, aggregations: "citation_count_aggregation", page: { size: 1, cursor: [] }).response.aggregations.citations.buckets - - + query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" + results = Event.query(query, doi:doi, aggregations: "citation_count_aggregation", page: { size: 1, cursor: [] }).response.aggregations.citations.buckets results.any? ? results.first.total.value : 0 end def citations(doi) doi.downcase.split(",").map do |item| - {id: item, total: EventsQuery.new.doi_citations(item)} + {id: item, count: EventsQuery.new.doi_citations(item)} end end diff --git a/spec/queries/events_query_spec.rb b/spec/queries/events_query_spec.rb index 7a535c22b..5646a0643 100644 --- a/spec/queries/events_query_spec.rb +++ b/spec/queries/events_query_spec.rb @@ -4,10 +4,18 @@ # before(:each) { allow(Time.zone).to receive(:now).and_return(Time.mktime(2015, 4, 8)) } context "citation events" do - subject { create(:event_for_datacite_related) } - + # subject { create(:event_for_datacite_related) } + let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } + let!(:event) { create_list(:event_for_datacite_related, 3,obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } + let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } + + before do + Event.import + sleep 1 + end + it "doi_citations" do - expect(EventsQuery.new.doi_citations(subject.obj_id.gsub("https://doi.org/",""))).to eq(1) + expect(EventsQuery.new.doi_citations("10.0260/co.2004960.v1")).to eq(1) end it "doi_citations wiht 0 citations" do @@ -15,7 +23,11 @@ end it "citations" do - expect(EventsQuery.new.citations("10.5061/dryad.47sd5/1,10.5061/dryad.47sd5/2,10.5061/dryad.47sd5/3")).to eq(1) + results = EventsQuery.new.citations("10.5061/dryad.47sd5/1,10.5061/dryad.47sd5/2,10.0260/co.2004960.v1") + citations = results.select { |item| item[:id] == "10.5061/dryad.47sd5/2" }.first + no_citations = results.select { |item| item[:id] == "10.5061/dryad.47sd5/1" }.first + expect(citations[:count]).to eq(3) + expect(no_citations[:count]).to eq(0) end end end diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index fe1136e62..ca03083c9 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -599,7 +599,7 @@ let!(:event) { create(:event_for_datacite_related) } let!(:events) { create_list(:event_for_datacite_related, 5, obj_id: event.subj_id) } let(:doi) { (event.subj_id).gsub("https://doi.org/","")} - let(:uri) { "/events?aggregations=citations_aggregations&doi=#{doi}" } + let(:uri) { "/events?doi=#{doi}" } before do Event.import @@ -673,7 +673,7 @@ let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } let(:doi) { (event.obj_id).gsub("https://doi.org/","")} - let(:uri) { "/events?aggregations=citations_aggregations&doi=#{doi}" } + let(:uri) { "/events?doi=#{doi}" } before do Event.import @@ -704,8 +704,8 @@ let!(:event) { create_list(:event_for_datacite_related, 50, relation_type_id: "is-cited-by") } let(:doi) { "https://doi.org/10.5061/dryad.47sd5/1".gsub("https://doi.org/","")} let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } - let(:dois) { ((event.map{ |e| e['subj_id'].gsub("https://doi.org/","")})[1, 20]).join(",")} - let(:uri) { "/events?aggregations=metrics_aggregations&doi=#{dois}" } + let(:dois) { ((event.map{ |e| e['obj_id'].gsub("https://doi.org/","")})[1, 20]).join(",")} + let(:uri) { "/events?doi=#{dois}" } before do Event.import @@ -720,18 +720,46 @@ it "json" do get uri, nil, headers - puts event[2].inspect - - expect(last_response.status).to eq(200) response = JSON.parse(last_response.body) - puts response.dig("meta", "uniqueCitations") citations = (response.dig("meta", "uniqueCitations")).select { |item| item["id"] == doi } total = response.dig("meta", "total") expect(total).to eq(51) expect((citations.select { |doi| dois.split(",").include?(doi["id"]) }).length).to eq(20) end + + # let(:uri) { "/events?doi=#{dois}" } + + # it "performance test" do + + + # expect { + # get uri, nil, headers + # }.to perform_under(2000).ms + + + # expect { + # get "/events", nil, headers + # }.to perform_under(1800).ms + + + # expect { + # get uri, nil, headers + # }.to perform_at_least(10).within(0.4).warmup(0.2).ips + + + # expect{ + # get("/events?doi=10.5061/dryad.47sd5/1", nil, {"HTTP_ACCEPT" => "application/vnd.api+json; version=2" }) + # }.to perform_slower_than.within(2).warmup(0.2) { + # get("/events", nil, {"HTTP_ACCEPT" => "application/vnd.api+json; version=2" }) + # }.once + + + + # end + + end # Just test that the API can be accessed without a token. From 522d8b629bcb96d2746f511083b899fdf8ca29da Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 26 Nov 2019 17:05:49 +0100 Subject: [PATCH 05/10] clean up comments --- app/controllers/events_controller.rb | 2 -- app/models/event.rb | 11 ---------- spec/queries/events_query_spec.rb | 2 -- spec/requests/events_spec.rb | 32 ---------------------------- 4 files changed, 47 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 1439df7f1..a021861d7 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -85,7 +85,6 @@ def index elsif params[:ids].present? response = Event.find_by_id(params[:ids], page: page, sort: sort) else - # params[:query] = '(subj_id:"https://doi.org/10.11646/zootaxa.4263.2.11" AND (relation_type_id:cites OR relation_type_id:is-supplement-to OR relation_type_id:references)) OR (obj_id:"https://doi.org/10.11646/zootaxa.4263.2.11" AND (relation_type_id:is-cited-by OR relation_type_id:is-supplemented-by OR relation_type_id:is-referenced-by))' response = Event.query(params[:query], subj_id: params[:subj_id], obj_id: params[:obj_id], @@ -125,7 +124,6 @@ def index dois_citations = total.positive? && aggregations.blank? || aggregations.include?("query_aggregations") ? facet_citations_by_year_v1(response.response.aggregations.dois_citations) : nil citations_histogram = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_year(response.response.aggregations.citations_histogram) : nil citations = params[:doi].present? ? EventsQuery.new.citations(params[:doi]) : [] - # citations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.citations.dois.buckets) : nil references = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.references.dois.buckets) : nil relations = total.positive? && params[:doi].present? && aggregations.include?("citations_aggregations") ? facet_citations_by_dois(response.response.aggregations.relations.dois.buckets) : nil views_histogram = total.positive? && aggregations.include?("metrics_aggregations") ? facet_counts_by_year_month(response.response.aggregations.views_histogram) : nil diff --git a/app/models/event.rb b/app/models/event.rb index 27e917919..9312e6a44 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -290,12 +290,6 @@ def self.citations_aggregations(doi) filter: citations_filter, aggs: { years: { histogram: { field: 'citation_year', interval: 1 , min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: 'total' }}}}, "sum_distribution" => sum_year_distribution } }, - citations: { - filter: citations_filter, - aggs: { dois: { - terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} - }} - }, references: { filter: references_filter, aggs: { dois: { @@ -314,16 +308,11 @@ def self.citations_aggregations(doi) def self.citation_count_aggregation(doi) - # doi = Event.new.normalize_doi(doi) if doi.present? - # citations_filter = { script: { script: "(#{PASSIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['subj_id'].value) || (#{ACTIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['obj_id'].value)" } } - # references_filter = { script: { script: "(#{PASSIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['obj_id'].value) || (#{ACTIVE_RELATION_TYPES}.contains(doc['relation_type_id'].value) && '#{doi}' == doc['subj_id'].value)" } } - { citations: { terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} } } - end diff --git a/spec/queries/events_query_spec.rb b/spec/queries/events_query_spec.rb index 5646a0643..31ab549e3 100644 --- a/spec/queries/events_query_spec.rb +++ b/spec/queries/events_query_spec.rb @@ -1,10 +1,8 @@ require 'rails_helper' describe EventsQuery, elasticsearch: true do - # before(:each) { allow(Time.zone).to receive(:now).and_return(Time.mktime(2015, 4, 8)) } context "citation events" do - # subject { create(:event_for_datacite_related) } let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } let!(:event) { create_list(:event_for_datacite_related, 3,obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index ca03083c9..6435d1999 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -728,38 +728,6 @@ expect(total).to eq(51) expect((citations.select { |doi| dois.split(",").include?(doi["id"]) }).length).to eq(20) end - - # let(:uri) { "/events?doi=#{dois}" } - - # it "performance test" do - - - # expect { - # get uri, nil, headers - # }.to perform_under(2000).ms - - - # expect { - # get "/events", nil, headers - # }.to perform_under(1800).ms - - - # expect { - # get uri, nil, headers - # }.to perform_at_least(10).within(0.4).warmup(0.2).ips - - - # expect{ - # get("/events?doi=10.5061/dryad.47sd5/1", nil, {"HTTP_ACCEPT" => "application/vnd.api+json; version=2" }) - # }.to perform_slower_than.within(2).warmup(0.2) { - # get("/events", nil, {"HTTP_ACCEPT" => "application/vnd.api+json; version=2" }) - # }.once - - - - # end - - end # Just test that the API can be accessed without a token. From ef27ccf804bd6e0ed11ea9a9b170ef9d2ccbc9fe Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 26 Nov 2019 19:17:00 +0100 Subject: [PATCH 06/10] rubocop fixes --- .solargraph.yml | 24 ++++++ app/models/event.rb | 130 +++++++++++++++--------------- app/queries/events_query.rb | 13 ++- spec/queries/events_query_spec.rb | 10 ++- spec/requests/events_spec.rb | 116 +++++++++++++------------- 5 files changed, 161 insertions(+), 132 deletions(-) create mode 100644 .solargraph.yml diff --git a/.solargraph.yml b/.solargraph.yml new file mode 100644 index 000000000..82ac1e5fb --- /dev/null +++ b/.solargraph.yml @@ -0,0 +1,24 @@ +--- +include: +- "**/*.rb" +exclude: +- spec/**/* +- test/**/* +- vendor/**/* +- ".bundle/**/*" +require: +- actioncable +- actionmailer +- actionpack +- actionview +- activejob +- activemodel +- activerecord +- activestorage +- activesupport +domains: [] +reporters: +- rubocop +- require_not_found +require_paths: [] +max_files: 5000 diff --git a/app/models/event.rb b/app/models/event.rb index 9312e6a44..c0fe36041 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class Event < ActiveRecord::Base # include helper module for query caching include Cacheable @@ -23,8 +25,8 @@ class Event < ActiveRecord::Base # include state machine include AASM - aasm :whiny_transitions => false do - state :waiting, :initial => true + aasm whiny_transitions: false do + state :waiting, initial: true state :working, :failed, :done # Reset after failure @@ -67,7 +69,7 @@ class Event < ActiveRecord::Base ] ACTIVE_RELATION_TYPES = [ - "cites", + "cites", "is-supplement-to", "references" ] @@ -139,7 +141,7 @@ class Event < ActiveRecord::Base indexes :access_method, type: :keyword indexes :metric_type, type: :keyword indexes :total, type: :integer - indexes :license, type: :text, fields: { keyword: { type: "keyword" }} + indexes :license, type: :text, fields: { keyword: { type: "keyword" } } indexes :error_messages, type: :object indexes :callback, type: :text indexes :aasm_state, type: :keyword @@ -154,7 +156,7 @@ class Event < ActiveRecord::Base indexes :cache_key, type: :keyword end - def as_indexed_json(options={}) + def as_indexed_json(options = {}) { "uuid" => uuid, "subj_id" => subj_id, @@ -195,10 +197,10 @@ def citation_id end def self.query_fields - ['subj_id^10', 'obj_id^10', 'subj.name^5', 'subj.author^5', 'subj.periodical^5', 'subj.publisher^5', 'obj.name^5', 'obj.author^5', 'obj.periodical^5', 'obj.publisher^5', '_all'] + ["subj_id^10", "obj_id^10", "subj.name^5", "subj.author^5", "subj.periodical^5", "subj.publisher^5", "obj.name^5", "obj.author^5", "obj.periodical^5", "obj.publisher^5", "_all"] end - def self.query_aggregations(doi=nil) + def self.query_aggregations(doi = nil) sum_distribution = { sum_bucket: { buckets_path: "year_months>total_by_year_month" @@ -212,17 +214,17 @@ def self.query_aggregations(doi=nil) } { - sources: { terms: { field: 'source_id', size: 50, min_doc_count: 1 } }, - prefixes: { terms: { field: 'prefix', size: 50, min_doc_count: 1 } }, - registrants: { terms: { field: 'registrant_id', size: 50, min_doc_count: 1 }, aggs: { year: { date_histogram: { field: 'occurred_at', interval: 'year', min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: 'total' }}}}} }, - pairings: { terms: { field: 'registrant_id', size: 50, min_doc_count: 1 }, aggs: { recipient: { terms: { field: 'registrant_id', size: 50, min_doc_count: 1 }, aggs: { "total" => { sum: { field: 'total' }}}}} }, - citation_types: { terms: { field: 'citation_type', size: 50, min_doc_count: 1 }, aggs: { year_months: { date_histogram: { field: 'occurred_at', interval: 'month', min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: 'total' }}}}} }, - relation_types: { terms: { field: 'relation_type_id', size: 50, min_doc_count: 1 }, aggs: { year_months: { date_histogram: { field: 'occurred_at', interval: 'month', min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: 'total' }}}},"sum_distribution"=>sum_distribution} }, - dois: { terms: { field: 'obj_id', size: 50, min_doc_count: 1 }, aggs: { relation_types: { terms: { field: 'relation_type_id',size: 50, min_doc_count: 1 }, aggs: { "total_by_type" => { sum: { field: 'total' }}}}} }, + sources: { terms: { field: "source_id", size: 50, min_doc_count: 1 } }, + prefixes: { terms: { field: "prefix", size: 50, min_doc_count: 1 } }, + registrants: { terms: { field: "registrant_id", size: 50, min_doc_count: 1 }, aggs: { year: { date_histogram: { field: "occurred_at", interval: "year", min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: "total" } } } } } }, + pairings: { terms: { field: "registrant_id", size: 50, min_doc_count: 1 }, aggs: { recipient: { terms: { field: "registrant_id", size: 50, min_doc_count: 1 }, aggs: { "total" => { sum: { field: "total" } } } } } }, + citation_types: { terms: { field: "citation_type", size: 50, min_doc_count: 1 }, aggs: { year_months: { date_histogram: { field: "occurred_at", interval: "month", min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: "total" } } } } } }, + relation_types: { terms: { field: "relation_type_id", size: 50, min_doc_count: 1 }, aggs: { year_months: { date_histogram: { field: "occurred_at", interval: "month", min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: "total" } } } }, "sum_distribution" => sum_distribution } }, + dois: { terms: { field: "obj_id", size: 50, min_doc_count: 1 }, aggs: { relation_types: { terms: { field: "relation_type_id", size: 50, min_doc_count: 1 }, aggs: { "total_by_type" => { sum: { field: "total" } } } } } }, dois_usage: { - filter: { script: { script: "doc['source_id'].value == 'datacite-usage' && doc['occurred_at'].size() > 0 && doc['obj.datePublished'].size() > 0 && doc['occurred_at'].value.getMillis() >= doc['obj.datePublished'].value.getMillis() && doc['occurred_at'].value.getMillis() < new Date().getTime()" }}, + filter: { script: { script: "doc['source_id'].value == 'datacite-usage' && doc['occurred_at'].size() > 0 && doc['obj.datePublished'].size() > 0 && doc['occurred_at'].value.getMillis() >= doc['obj.datePublished'].value.getMillis() && doc['occurred_at'].value.getMillis() < new Date().getTime()" } }, aggs: { - dois: { terms: { field: 'obj_id', size: 50, min_doc_count: 1 }, aggs: { relation_types: { terms: { field: 'relation_type_id',size: 50, min_doc_count: 1 }, aggs: { "total_by_type" => { sum: { field: 'total' }}}}} } } + dois: { terms: { field: "obj_id", size: 50, min_doc_count: 1 }, aggs: { relation_types: { terms: { field: "relation_type_id", size: 50, min_doc_count: 1 }, aggs: { "total_by_type" => { sum: { field: "total" } } } } } } } }, dois_citations: { filter: { @@ -230,7 +232,7 @@ def self.query_aggregations(doi=nil) script: "#{INCLUDED_RELATION_TYPES}.contains(doc['relation_type_id'].value)" } }, - aggs: { years: { date_histogram: { field: 'occurred_at', interval: 'year', min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: 'total' }}}},"sum_distribution"=>sum_year_distribution} + aggs: { years: { date_histogram: { field: "occurred_at", interval: "year", min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: "total" } } } }, "sum_distribution" => sum_year_distribution } } } end @@ -241,7 +243,7 @@ def self.metrics_aggregations(doi = nil) buckets_path: "year_months>total_by_year_month" } } - + views_filter = { script: { script: "doc['relation_type_id'].value == 'unique-dataset-investigations-regular' && doc['source_id'].value == 'datacite-usage' && doc['occurred_at'].size() > 0 && doc['obj.datePublished'].size() > 0 && doc['occurred_at'].value.getMillis() >= doc['obj.datePublished'].value.getMillis() && doc['occurred_at'].value.getMillis() < new Date().getTime()" } } downloads_filter = { script: { script: "doc['relation_type_id'].value == 'unique-dataset-requests-regular' && doc['source_id'].value == 'datacite-usage' && doc['occurred_at'].size() > 0 && doc['obj.datePublished'].size() > 0 && doc['occurred_at'].value.getMillis() >= doc['obj.datePublished'].value.getMillis() && doc['occurred_at'].value.getMillis() < new Date().getTime()" } } @@ -249,31 +251,31 @@ def self.metrics_aggregations(doi = nil) views: { filter: views_filter, aggs: { dois: { - terms: { field: 'obj_id', size: 50, min_doc_count: 1 } , aggs: { "total_by_type" => { sum: { field: 'total' }}} - }} + terms: { field: "obj_id", size: 50, min_doc_count: 1 } , aggs: { "total_by_type" => { sum: { field: "total" } } } + } } }, views_histogram: { filter: views_filter, aggs: { - year_months: { date_histogram: { field: 'occurred_at', interval: 'month', min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: 'total' } } } }, "sum_distribution" => sum_distribution + year_months: { date_histogram: { field: "occurred_at", interval: "month", min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: "total" } } } }, "sum_distribution" => sum_distribution } }, downloads: { filter: downloads_filter, aggs: { dois: { - terms: { field: 'obj_id', size: 50, min_doc_count: 1} , aggs: { "total_by_type" => { sum: { field: 'total' }}} - }} + terms: { field: "obj_id", size: 50, min_doc_count: 1 } , aggs: { "total_by_type" => { sum: { field: "total" } } } + } } }, downloads_histogram: { filter: downloads_filter, aggs: { - year_months: { date_histogram: { field: 'occurred_at', interval: 'month', min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: 'total' } } } }, "sum_distribution" => sum_distribution + year_months: { date_histogram: { field: "occurred_at", interval: "month", min_doc_count: 1 }, aggs: { "total_by_year_month" => { sum: { field: "total" } } } }, "sum_distribution" => sum_distribution } } } end - def self.citations_aggregations(doi) + def self.citations_aggregations(doi) doi = Event.new.normalize_doi(doi) if doi.present? sum_year_distribution = { @@ -288,49 +290,49 @@ def self.citations_aggregations(doi) { citations_histogram: { filter: citations_filter, - aggs: { years: { histogram: { field: 'citation_year', interval: 1 , min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: 'total' }}}}, "sum_distribution" => sum_year_distribution } + aggs: { years: { histogram: { field: "citation_year", interval: 1 , min_doc_count: 1 }, aggs: { "total_by_year" => { sum: { field: "total" } } } }, "sum_distribution" => sum_year_distribution } }, references: { filter: references_filter, aggs: { dois: { - terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} - }} + terms: { field: "doi", size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: "citation_id" } } } + } } }, relations: { filter: { script: { script: "#{RELATIONS_RELATION_TYPES}.contains(doc['relation_type_id'].value)" } }, aggs: { dois: { - terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} - }} + terms: { field: "doi", size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: "citation_id" } } } + } } } - } + } end def self.citation_count_aggregation(doi) - { + { citations: { - terms: { field: 'doi', size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: 'citation_id' }}} + terms: { field: "doi", size: 100, min_doc_count: 1 }, aggs: { total: { cardinality: { field: "citation_id" } } } } } end - - def self.advanced_aggregations(doi=nil) + + def self.advanced_aggregations(doi = nil) { - unique_obj_count: { cardinality: { field: 'obj_id' }}, - unique_subj_count: { cardinality: { field: 'subj_id' }} + unique_obj_count: { cardinality: { field: "obj_id" } }, + unique_subj_count: { cardinality: { field: "subj_id" } } } end # return results for one or more ids - def self.find_by_id(ids, options={}) + def self.find_by_id(ids, options = {}) ids = ids.split(",") if ids.is_a?(String) options[:page] ||= {} options[:page][:number] ||= 1 options[:page][:size] ||= 1000 - options[:sort] ||= { created_at: { order: "asc" }} + options[:sort] ||= { created_at: { order: "asc" } } __elasticsearch__.search({ from: (options.dig(:page, :number) - 1) * options.dig(:page, :size), @@ -345,7 +347,7 @@ def self.find_by_id(ids, options={}) }) end - def self.import_by_ids(options={}) + def self.import_by_ids(options = {}) from_id = (options[:from_id] || Event.minimum(:id)).to_i until_id = (options[:until_id] || Event.maximum(:id)).to_i @@ -356,7 +358,7 @@ def self.import_by_ids(options={}) end end - def self.import_by_id(options={}) + def self.import_by_id(options = {}) return nil unless options[:id].present? id = options[:id].to_i @@ -373,8 +375,8 @@ def self.import_by_id(options={}) body: events.map { |event| { index: { _id: event.id, data: event.as_indexed_json } } } # log errors - errors += response['items'].map { |k, v| k.values.first['error'] }.compact.length - response['items'].select { |k, v| k.values.first['error'].present? }.each do |err| + errors += response["items"].map { |k, v| k.values.first["error"] }.compact.length + response["items"].select { |k, v| k.values.first["error"].present? }.each do |err| logger.error "[Elasticsearch] " + err.inspect end @@ -399,7 +401,7 @@ def self.import_by_id(options={}) logger.info "[Elasticsearch] Imported #{count} events with IDs #{id} - #{(id + 499)}." end - def self.update_crossref(options={}) + def self.update_crossref(options = {}) logger = Logger.new(STDOUT) size = (options[:size] || 1000).to_i @@ -425,39 +427,39 @@ def self.update_crossref(options={}) response.results.total end - def self.update_datacite_crossref(options={}) + def self.update_datacite_crossref(options = {}) update_datacite_ra(options.merge(ra: "crossref")) end - def self.update_datacite_medra(options={}) + def self.update_datacite_medra(options = {}) update_datacite_ra(options.merge(ra: "medra")) end - def self.update_datacite_kisti(options={}) + def self.update_datacite_kisti(options = {}) update_datacite_ra(options.merge(ra: "kisti")) end - def self.update_datacite_jalc(options={}) + def self.update_datacite_jalc(options = {}) update_datacite_ra(options.merge(ra: "jalc")) end - def self.update_datacite_op(options={}) + def self.update_datacite_op(options = {}) update_datacite_ra(options.merge(ra: "op")) end - def self.update_datacite_medra(options={}) + def self.update_datacite_medra(options = {}) update_datacite_ra(options.merge(ra: "medra")) end - def self.update_datacite_medra(options={}) + def self.update_datacite_medra(options = {}) update_datacite_ra(options.merge(ra: "medra")) end - def self.update_datacite_medra(options={}) + def self.update_datacite_medra(options = {}) update_datacite_ra(options.merge(ra: "medra")) end - def self.update_datacite_ra(options={}) + def self.update_datacite_ra(options = {}) logger = Logger.new(STDOUT) size = (options[:size] || 1000).to_i @@ -487,7 +489,7 @@ def self.update_datacite_ra(options={}) response.results.total end - def self.update_registrant(options={}) + def self.update_registrant(options = {}) logger = Logger.new(STDOUT) size = (options[:size] || 1000).to_i @@ -518,7 +520,7 @@ def self.update_registrant(options={}) response.results.total end - def self.update_datacite_orcid_auto_update(options={}) + def self.update_datacite_orcid_auto_update(options = {}) logger = Logger.new(STDOUT) size = (options[:size] || 1000).to_i @@ -557,8 +559,8 @@ def send_callback "messageAction" => message_action, "sourceToken" => source_token, "total" => total, - "timestamp" => timestamp }} - Maremma.post(callback, data: data.to_json, token: ENV['API_KEY']) + "timestamp" => timestamp } } + Maremma.post(callback, data: data.to_json, token: ENV["API_KEY"]) end def access_method @@ -583,7 +585,7 @@ def doi end def prefix - [doi.map { |d| d.to_s.split('/', 2).first }].compact + [doi.map { |d| d.to_s.split("/", 2).first }].compact end def orcid @@ -616,7 +618,7 @@ def citation_type def doi_from_url(url) if /\A(?:(http|https):\/\/(dx\.)?(doi.org|handle.test.datacite.org)\/)?(doi:)?(10\.\d{4,5}\/.+)\z/.match(url) uri = Addressable::URI.parse(url) - uri.path.gsub(/^\//, '').downcase + uri.path.gsub(/^\//, "").downcase end end @@ -648,16 +650,16 @@ def obj_cache_key end def citation_year - "" unless (INCLUDED_RELATION_TYPES+RELATIONS_RELATION_TYPES).include?(relation_type_id) - subj_publication = subj['date_published'] || (date_published(subj_id) || year_month) - obj_publication = obj['date_published'] || (date_published(obj_id) || year_month) + "" unless (INCLUDED_RELATION_TYPES + RELATIONS_RELATION_TYPES).include?(relation_type_id) + subj_publication = subj["date_published"] || (date_published(subj_id) || year_month) + obj_publication = obj["date_published"] || (date_published(obj_id) || year_month) [subj_publication[0..3].to_i, obj_publication[0..3].to_i].max end def date_published(doi) - ## TODO: we need to make sure all the dois from other RA are indexed + ## TODO: we need to make sure all the dois from other RA are indexed item = Doi.where(doi: doi_from_url(doi)).first - item[:publication_year].to_s if item.present? + item[:publication_year].to_s if item.present? end def set_defaults diff --git a/app/queries/events_query.rb b/app/queries/events_query.rb index ca24fd39e..cc63e6e7a 100644 --- a/app/queries/events_query.rb +++ b/app/queries/events_query.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + class EventsQuery attr_reader :relation include Facetable ACTIVE_RELATION_TYPES = [ - "cites", + "cites", "is-supplement-to", "references" ] @@ -21,19 +23,16 @@ def initialize(relation = Event.new) def doi_citations(doi) pid = relation.normalize_doi(doi) - query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" + query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" results = Event.query(query, doi:doi, aggregations: "citation_count_aggregation", page: { size: 1, cursor: [] }).response.aggregations.citations.buckets results.any? ? results.first.total.value : 0 end def citations(doi) doi.downcase.split(",").map do |item| - {id: item, count: EventsQuery.new.doi_citations(item)} + { id: item, count: EventsQuery.new.doi_citations(item) } end end - - # private - -end \ No newline at end of file +end diff --git a/spec/queries/events_query_spec.rb b/spec/queries/events_query_spec.rb index 31ab549e3..80f18078f 100644 --- a/spec/queries/events_query_spec.rb +++ b/spec/queries/events_query_spec.rb @@ -1,17 +1,19 @@ -require 'rails_helper' +# frozen_string_literal: true + +require "rails_helper" describe EventsQuery, elasticsearch: true do context "citation events" do let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } - let!(:event) { create_list(:event_for_datacite_related, 3,obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } + let!(:event) { create_list(:event_for_datacite_related, 3, obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } - + before do Event.import sleep 1 end - + it "doi_citations" do expect(EventsQuery.new.doi_citations("10.0260/co.2004960.v1")).to eq(1) end diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index 6435d1999..fad36576b 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -1,6 +1,8 @@ +# frozen_string_literal: true + require "rails_helper" -require 'pp' - +require "pp" + describe "/events", type: :request, elasticsearch: true do before(:each) do @@ -9,36 +11,36 @@ end let(:event) { build(:event) } - let(:errors) { [{ "status" => "401", "title"=>"Bad credentials."}] } + let(:errors) { [{ "status" => "401", "title" => "Bad credentials." }] } # Successful response from creating via the API. let(:success) { { "id" => event.uuid, "type" => "events", - "attributes"=>{ + "attributes" => { "subjId" => "http://www.citeulike.org/user/dbogartoit", "objId" => "http://doi.org/10.1371/journal.pmed.0030186", - "messageAction"=>"create", - "sourceToken"=>"citeulike_123", - "relationTypeId"=>"bookmarks", - "sourceId"=>"citeulike", - "total"=>1, - "license"=>"https://creativecommons.org/publicdomain/zero/1.0/", - "occurredAt"=>"2015-04-08T00:00:00.000Z", - "subj"=> {"@id"=>"http://www.citeulike.org/user/dbogartoit", - "@type"=>"CreativeWork", - "author"=>[{"givenName"=>"dbogartoit"}], - "name"=>"CiteULike bookmarks for user dbogartoit", - "publisher"=> { "@type" => "Organization", "name" => "CiteULike" }, - "periodical"=> { "@type" => "Periodical", "@id" => "https://doi.org/10.13039/100011326", "name" => "CiteULike", "issn" => "9812-847X" }, - "funder"=> { "@type" => "Organization", "@id" => "https://doi.org/10.13039/100011326", "name" => "CiteULike" }, + "messageAction" => "create", + "sourceToken" => "citeulike_123", + "relationTypeId" => "bookmarks", + "sourceId" => "citeulike", + "total" => 1, + "license" => "https://creativecommons.org/publicdomain/zero/1.0/", + "occurredAt" => "2015-04-08T00:00:00.000Z", + "subj" => { "@id" => "http://www.citeulike.org/user/dbogartoit", + "@type" => "CreativeWork", + "author" => [{ "givenName" => "dbogartoit" }], + "name" => "CiteULike bookmarks for user dbogartoit", + "publisher" => { "@type" => "Organization", "name" => "CiteULike" }, + "periodical" => { "@type" => "Periodical", "@id" => "https://doi.org/10.13039/100011326", "name" => "CiteULike", "issn" => "9812-847X" }, + "funder" => { "@type" => "Organization", "@id" => "https://doi.org/10.13039/100011326", "name" => "CiteULike" }, "version" => "1.0", "proxyIdentifiers" => ["10.13039/100011326"], - "datePublished"=>"2006-06-13T16:14:19Z", - "dateModified"=>"2006-06-13T16:14:19Z", - "url"=>"http://www.citeulike.org/user/dbogartoit" + "datePublished" => "2006-06-13T16:14:19Z", + "dateModified" => "2006-06-13T16:14:19Z", + "url" => "http://www.citeulike.org/user/dbogartoit" }, - "obj"=>{} - }}} + "obj" => {} + } }} let(:token) { User.generate_token(role_id: "staff_admin") } let(:uuid) { SecureRandom.uuid } @@ -73,7 +75,7 @@ end context "with very long url" do - let(:url) {"http://navigator.eumetsat.int/soapservices/cswstartup?service=csw&version=2.0.2&request=getrecordbyid&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd&id=eo%3Aeum%3Adat%3Amult%3Arac-m11-iasia"} + let(:url) { "http://navigator.eumetsat.int/soapservices/cswstartup?service=csw&version=2.0.2&request=getrecordbyid&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd&id=eo%3Aeum%3Adat%3Amult%3Arac-m11-iasia" } let(:params) do { "data" => { "type" => "events", "attributes" => { @@ -102,7 +104,7 @@ post uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_nil end end @@ -114,7 +116,7 @@ post uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_blank end end @@ -132,7 +134,7 @@ post uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Source token can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Source token can't be blank" }]) expect(json["data"]).to be_nil end end @@ -150,7 +152,7 @@ post uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Source can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Source can't be blank" }]) expect(json["data"]).to be_blank end end @@ -168,7 +170,7 @@ post uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Subj can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Subj can't be blank" }]) expect(json["data"]).to be_blank end end @@ -238,8 +240,8 @@ { "data" => { "type" => "events", "attributes" => { "subjId" => "https://doi.org/10.18713/jimis-170117-1-2", - "subj" => {"@id":"https://doi.org/10.18713/jimis-170117-1-2","@type":"ScholarlyArticle","datePublished":"2017","proxyIdentifiers":[],"registrantId":"datacite.inist.umr7300"}, - "obj" => {"@id":"https://doi.org/10.1016/j.jastp.2013.05.001","@type":"ScholarlyArticle","datePublished":"2013-09","proxyIdentifiers":["13646826"],"registrantId":"datacite.crossref.citations"}, + "subj" => { "@id":"https://doi.org/10.18713/jimis-170117-1-2", "@type":"ScholarlyArticle", "datePublished":"2017", "proxyIdentifiers":[], "registrantId":"datacite.inist.umr7300" }, + "obj" => { "@id":"https://doi.org/10.1016/j.jastp.2013.05.001", "@type":"ScholarlyArticle", "datePublished":"2013-09", "proxyIdentifiers":["13646826"], "registrantId":"datacite.crossref.citations" }, "objId" => "https://doi.org/10.1016/j.jastp.2013.05.001", "relationTypeId" => "references", "sourceId" => "datacite-crossref", @@ -248,7 +250,7 @@ it "has registrant aggregation" do post uri, params, headers - + expect(last_response.status).to eq(201) expect(json["errors"]).to be_nil @@ -259,21 +261,21 @@ sleep 1 get uri, nil, headers - expect(json.dig("meta", "registrants",0,"count")).to eq(1) - expect(json.dig("meta", "registrants",0,"id")).to eq("datacite.crossref.citations") + expect(json.dig("meta", "registrants", 0, "count")).to eq(1) + expect(json.dig("meta", "registrants", 0, "id")).to eq("datacite.crossref.citations") end it "has citationsHistogram aggregation with correct citation year" do post uri, params, headers - + expect(last_response.status).to eq(201) expect(json["errors"]).to be_nil - + Event.import sleep 1 - get uri+"?aggregations=citations_aggregations&doi=10.1016/j.jastp.2013.05.001", nil, headers + get uri + "?aggregations=citations_aggregations&doi=10.1016/j.jastp.2013.05.001", nil, headers puts json.dig("meta", "citationsHistogram") - expect(json.dig("meta", "citationsHistogram","years",0,"title")).to eq("2017") + expect(json.dig("meta", "citationsHistogram", "years", 0, "title")).to eq("2017") end end end @@ -310,7 +312,7 @@ put uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_nil end end @@ -322,7 +324,7 @@ put uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_blank end end @@ -340,7 +342,7 @@ put uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Source token can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Source token can't be blank" }]) expect(json["data"]).to be_nil end end @@ -358,7 +360,7 @@ put uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Source can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Source can't be blank" }]) expect(json["data"]).to be_blank end end @@ -376,7 +378,7 @@ put uri, params, headers expect(last_response.status).to eq(422) - expect(json["errors"]).to eq([{"status"=>422, "title"=>"Subj can't be blank"}]) + expect(json["errors"]).to eq([{ "status" => 422, "title" => "Subj can't be blank" }]) expect(json["data"]).to be_blank end end @@ -474,7 +476,7 @@ put uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_nil end end @@ -486,7 +488,7 @@ put uri, params, headers expect(last_response.status).to eq(403) - expect(json["errors"]).to eq([{"status"=>"403", "title"=>"You are not authorized to access this resource."}]) + expect(json["errors"]).to eq([{ "status" => "403", "title" => "You are not authorized to access this resource." }]) expect(json["data"]).to be_blank end end @@ -516,7 +518,7 @@ it "JSON" do put uri, params, headers - + expect(last_response.status).to eq(422) expect(json.dig("errors", 0, "title")).to start_with("Invalid payload") expect(json["data"]).to be_blank @@ -585,7 +587,7 @@ get uri, nil, headers expect(last_response.status).to eq(404) - expect(json["errors"]).to eq([{"status"=>"404", "title"=>"The resource you are looking for doesn't exist."}]) + expect(json["errors"]).to eq([{ "status" => "404", "title" => "The resource you are looking for doesn't exist." }]) expect(json["data"]).to be_nil end end @@ -594,11 +596,11 @@ context "index" do # # let!(:event) { create(:event) } # # let(:uri) { "/events" } - + context "check meta unique" do let!(:event) { create(:event_for_datacite_related) } let!(:events) { create_list(:event_for_datacite_related, 5, obj_id: event.subj_id) } - let(:doi) { (event.subj_id).gsub("https://doi.org/","")} + let(:doi) { (event.subj_id).gsub("https://doi.org/", "") } let(:uri) { "/events?doi=#{doi}" } before do @@ -619,7 +621,7 @@ citations = (response.dig("meta", "uniqueCitations")).select { |item| item["id"] == doi } total = response.dig("meta", "total") - + expect(total).to eq(6) # puts citations.dig(:count) expect(citations.first["count"]).to eq(5) @@ -633,9 +635,9 @@ let!(:event2) { create(:event_for_datacite_related, obj_id: event.subj_id) } let!(:event3) { create(:event_for_datacite_related, obj_id: event.subj_id) } let!(:event4) { create(:event_for_datacite_related, obj_id: event.subj_id, relation_type_id:"has-part") } - let(:doi) { (event.subj_id).gsub("https://doi.org/","")} + let(:doi) { (event.subj_id).gsub("https://doi.org/", "") } let(:uri) { "/events?aggregations=citations_aggregations&doi=#{doi}" } - + before do Event.import sleep 1 @@ -658,7 +660,7 @@ relations = (response.dig("meta", "relations")).select { |item| item["id"] == doi } total = response.dig("meta", "total") - expect(json.dig("meta", "citationsHistogram","years",0,"title")).to eq("2015") + expect(json.dig("meta", "citationsHistogram", "years", 0, "title")).to eq("2015") expect(total).to eq(5) expect(citations.first["count"]).to eq(2) expect(citations.first["id"]).to eq(doi) @@ -672,7 +674,7 @@ context "check meta duplicated" do let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } - let(:doi) { (event.obj_id).gsub("https://doi.org/","")} + let(:doi) { (event.obj_id).gsub("https://doi.org/", "") } let(:uri) { "/events?doi=#{doi}" } before do @@ -702,9 +704,9 @@ # We cannot do this anymore as the aggregation needs to be done per doi context "unique citations for a list of dois" do let!(:event) { create_list(:event_for_datacite_related, 50, relation_type_id: "is-cited-by") } - let(:doi) { "https://doi.org/10.5061/dryad.47sd5/1".gsub("https://doi.org/","")} + let(:doi) { "https://doi.org/10.5061/dryad.47sd5/1".gsub("https://doi.org/", "") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } - let(:dois) { ((event.map{ |e| e['obj_id'].gsub("https://doi.org/","")})[1, 20]).join(",")} + let(:dois) { ((event.map{ |e| e["obj_id"].gsub("https://doi.org/", "") })[1, 20]).join(",") } let(:uri) { "/events?doi=#{dois}" } before do @@ -962,7 +964,7 @@ delete uri, nil, headers expect(last_response.status).to eq(404) - expect(json["errors"]).to eq([{"status"=>"404", "title"=>"The resource you are looking for doesn't exist."}]) + expect(json["errors"]).to eq([{ "status" => "404", "title" => "The resource you are looking for doesn't exist." }]) expect(json["data"]).to be_nil end end From d875b92346f24361d95e8ab445a326ab83dc254f Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 26 Nov 2019 19:19:46 +0100 Subject: [PATCH 07/10] spaces --- app/queries/events_query.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/queries/events_query.rb b/app/queries/events_query.rb index cc63e6e7a..a5d22cba9 100644 --- a/app/queries/events_query.rb +++ b/app/queries/events_query.rb @@ -33,6 +33,4 @@ def citations(doi) { id: item, count: EventsQuery.new.doi_citations(item) } end end - - end From f11839696051d6463712022a023416085077a3a1 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Thu, 28 Nov 2019 07:11:08 +0100 Subject: [PATCH 08/10] change variable typo --- spec/queries/events_query_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/queries/events_query_spec.rb b/spec/queries/events_query_spec.rb index 80f18078f..4776cb637 100644 --- a/spec/queries/events_query_spec.rb +++ b/spec/queries/events_query_spec.rb @@ -6,7 +6,7 @@ context "citation events" do let!(:event) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1") } - let!(:event) { create_list(:event_for_datacite_related, 3, obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } + let!(:event_references) { create_list(:event_for_datacite_related, 3, obj_id:"10.5061/dryad.47sd5/2", relation_type_id: "references") } let!(:copies) { create(:event_for_datacite_related, subj_id:"http://doi.org/10.0260/co.2004960.v2", obj_id:"http://doi.org/10.0260/co.2004960.v1", relation_type_id: "cites") } before do From 225bad9cd39808a2e33212cfd015bdd9b8adda3e Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Thu, 28 Nov 2019 07:16:50 +0100 Subject: [PATCH 09/10] remove relation --- app/queries/events_query.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/queries/events_query.rb b/app/queries/events_query.rb index a5d22cba9..4c111d248 100644 --- a/app/queries/events_query.rb +++ b/app/queries/events_query.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class EventsQuery - attr_reader :relation include Facetable @@ -17,12 +16,11 @@ class EventsQuery "is-referenced-by" ] - def initialize(relation = Event.new) - @relation = relation + def initialize() end def doi_citations(doi) - pid = relation.normalize_doi(doi) + pid = Event.new.normalize_doi(doi) query = "(subj_id:\"#{pid}\" AND (relation_type_id:#{PASSIVE_RELATION_TYPES.join(' OR relation_type_id:')})) OR (obj_id:\"#{pid}\" AND (relation_type_id:#{ACTIVE_RELATION_TYPES.join(' OR relation_type_id:')}))" results = Event.query(query, doi:doi, aggregations: "citation_count_aggregation", page: { size: 1, cursor: [] }).response.aggregations.citations.buckets results.any? ? results.first.total.value : 0 From 20cbac3d42baa8e16cfcdbe785eb55f1acfa0e41 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Thu, 28 Nov 2019 11:35:35 +0100 Subject: [PATCH 10/10] remove solar graph --- .gitignore | 1 + .solargraph.yml | 24 ------------------------ 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 .solargraph.yml diff --git a/.gitignore b/.gitignore index 91c1f7c5d..bd4cf6255 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ doc/dependencies* docker-compose.override.yml .ruby-version .vscode +.solargraph diff --git a/.solargraph.yml b/.solargraph.yml deleted file mode 100644 index 82ac1e5fb..000000000 --- a/.solargraph.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -include: -- "**/*.rb" -exclude: -- spec/**/* -- test/**/* -- vendor/**/* -- ".bundle/**/*" -require: -- actioncable -- actionmailer -- actionpack -- actionview -- activejob -- activemodel -- activerecord -- activestorage -- activesupport -domains: [] -reporters: -- rubocop -- require_not_found -require_paths: [] -max_files: 5000