diff --git a/Gemfile.lock b/Gemfile.lock index 7d0e9a095..9fa08cb5e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM latex-decode (~> 0.0) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) - bolognese (1.2.5) + bolognese (1.2.7) activesupport (>= 4.2.5, < 6) benchmark_methods (~> 0.7) bibtex-ruby (~> 4.1) @@ -214,7 +214,7 @@ GEM factory_bot_rails (4.11.1) factory_bot (~> 4.11.1) railties (>= 3.0.0) - faker (1.9.5) + faker (1.9.6) i18n (>= 0.7) faraday (0.14.0) multipart-post (>= 1.2, < 3) @@ -307,7 +307,7 @@ GEM rest-client (~> 2.0.2) marcel (0.3.3) mimemagic (~> 0.3.2) - maremma (4.2.3) + maremma (4.2.6) activesupport (>= 4.2.5, < 6) addressable (>= 2.3.6) builder (~> 3.2, >= 3.2.2) diff --git a/app/jobs/url_job.rb b/app/jobs/url_job.rb index 03bb95ca3..875178879 100644 --- a/app/jobs/url_job.rb +++ b/app/jobs/url_job.rb @@ -14,13 +14,13 @@ def perform(doi_id) response = Doi.get_doi(doi: doi.doi, agency: doi.agency) url = response.body.dig('data', 'values', 0, 'data', 'value') if url.present? - if (doi.is_registered_or_findable? || %w(europ crossref medra).include?(doi.provider_id)) && doi.minted.blank? + if (doi.is_registered_or_findable? || %w(europ crossref medra kisti jalc op).include?(doi.provider_id)) && doi.minted.blank? doi.update_attributes(url: url, minted: Time.zone.now) else doi.update_attributes(url: url) end - doi.update_attributes(aasm_state: "findable") if %w(europ crossref medra).include?(doi.provider_id) + doi.update_attributes(aasm_state: "findable") if %w(europ crossref medra kisti jalc op).include?(doi.provider_id) doi.__elasticsearch__.index_document diff --git a/app/models/ability.rb b/app/models/ability.rb index c77272a53..ac59b48dd 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -12,7 +12,7 @@ def initialize(user) if user.role_id == "staff_admin" can :manage, :all cannot [:new, :create], Doi do |doi| - doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || doi.client.symbol.downcase.start_with?("crossref.") || doi.client.symbol.downcase.start_with?("medra.")) + doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first)) end elsif user.role_id == "staff_user" can :read, :all @@ -65,7 +65,7 @@ def initialize(user) can [:read, :destroy, :update, :register_url, :validate, :undo, :get_url, :get_urls, :read_landing_page_results], Doi, :client_id => user.client_id can [:new, :create], Doi do |doi| - doi.client.prefixes.where(prefix: doi.prefix).present? || doi.client.symbol.downcase.start_with?("crossref.") || doi.client.symbol.downcase.start_with?("medra.") + doi.client.prefixes.where(prefix: doi.prefix).present? || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first) end can [:read], Doi do |doi| doi.findable? diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index 4b9558eb7..0dee6a17b 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -9,7 +9,7 @@ module Indexable IndexJob.perform_later(self) if self.class.name == "Doi" update_column(:indexed, Time.zone.now) - send_import_message(self.to_jsonapi) if aasm_state == "findable" unless (Rails.env.test? || client_id == "crossref.citations" || client_id == "medra.citations") + send_import_message(self.to_jsonapi) if aasm_state == "findable" unless (Rails.env.test? || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first)) end end @@ -183,12 +183,12 @@ def query(query, options={}) must << { terms: { "software.raw" => options[:software].split(",") }} if options[:software].present? must << { term: { repository_id: options[:repository_id].gsub("/", '\/') }} if options[:repository_id].present? must_not << { exists: { field: "deleted_at" }} unless options[:include_deleted] - must_not << { terms: { provider_id: ["crossref", "medra"] }} if options[:exclude_registration_agencies] + must_not << { terms: { provider_id: ["crossref", "medra", "op"] }} if options[:exclude_registration_agencies] elsif self.name == "Doi" must << { terms: { aasm_state: options[:state].to_s.split(",") }} if options[:state].present? must << { range: { registered: { gte: "#{options[:registered].split(",").min}||/y", lte: "#{options[:registered].split(",").max}||/y", format: "yyyy" }}} if options[:registered].present? must << { term: { "client.repository_id" => options[:repository_id].upcase.gsub("/", '\/') }} if options[:repository_id].present? - must_not << { terms: { provider_id: ["crossref", "medra"] }} if options[:exclude_registration_agencies] + must_not << { terms: { provider_id: ["crossref", "medra", "op"] }} if options[:exclude_registration_agencies] elsif self.name == "Event" must << { term: { subj_id: options[:subj_id] }} if options[:subj_id].present? must << { term: { obj_id: options[:obj_id] }} if options[:obj_id].present? diff --git a/app/models/doi.rb b/app/models/doi.rb index 0beeb2e76..5a8946f0c 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -580,7 +580,7 @@ def registerable? # end def is_registered_or_findable? - %w(registered findable).include?(aasm_state) || %w(crossref medra).include?(provider_id) + %w(registered findable).include?(aasm_state) || %w(crossref medra op).include?(provider_id) end def validatable? diff --git a/app/models/event.rb b/app/models/event.rb index 0425766c0..c0d9d0bab 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -306,47 +306,55 @@ def self.update_crossref(options={}) end def self.update_datacite_crossref(options={}) - logger = Logger.new(STDOUT) + update_datacite_ra(options.merge(ra: "crossref")) + end - size = (options[:size] || 1000).to_i - cursor = (options[:cursor] || 0).to_i + def self.update_datacite_medra(options={}) + update_datacite_ra(options.merge(ra: "medra")) + end - response = Event.query(nil, source_id: "datacite-crossref", page: { size: 1, cursor: cursor }) - logger.info "[Update] #{response.results.total} events for source datacite-crossref." + def self.update_datacite_kisti(options={}) + update_datacite_ra(options.merge(ra: "kisti")) + end - # walk through results using cursor - if response.results.total > 0 - while response.results.results.length > 0 do - response = Event.query(nil, source_id: "datacite-crossref", page: { size: size, cursor: cursor }) - break unless response.results.results.length > 0 + def self.update_datacite_jalc(options={}) + update_datacite_ra(options.merge(ra: "jalc")) + end - logger.info "[Update] Updating #{response.results.results.length} datacite-crossref events starting with _id #{cursor + 1}." - cursor = response.results.to_a.last[:sort].first.to_i + def self.update_datacite_op(options={}) + update_datacite_ra(options.merge(ra: "op")) + end - dois = response.results.results.map(&:obj_id).uniq - CrossrefDoiJob.perform_later(dois, options) - end - end + def self.update_datacite_medra(options={}) + update_datacite_ra(options.merge(ra: "medra")) + end - response.results.total + def self.update_datacite_medra(options={}) + update_datacite_ra(options.merge(ra: "medra")) end def self.update_datacite_medra(options={}) + update_datacite_ra(options.merge(ra: "medra")) + end + + def self.update_datacite_ra(options={}) logger = Logger.new(STDOUT) size = (options[:size] || 1000).to_i cursor = (options[:cursor] || 0).to_i + ra = options[:ra] || "crossref" + source_id = "datacite-#{ra}" - response = Event.query(nil, source_id: "datacite-medra", page: { size: 1, cursor: cursor }) - logger.info "[Update] #{response.results.total} events for source datacite-medra." + response = Event.query(nil, source_id: source_id, page: { size: 1, cursor: cursor }) + logger.info "[Update] #{response.results.total} events for source #{source_id}." # walk through results using cursor if response.results.total > 0 while response.results.results.length > 0 do - response = Event.query(nil, source_id: "datacite-medra", page: { size: size, cursor: cursor }) + response = Event.query(nil, source_id: source_id, page: { size: size, cursor: cursor }) break unless response.results.results.length > 0 - logger.info "[Update] Updating #{response.results.results.length} datacite-medra events starting with _id #{cursor + 1}." + logger.info "[Update] Updating #{response.results.results.length} #{source_id} events starting with _id #{cursor + 1}." cursor = response.results.to_a.last[:sort].first.to_i dois = response.results.results.map(&:obj_id).uniq diff --git a/lib/tasks/event.rake b/lib/tasks/event.rake index f18992ad5..f67a23a22 100644 --- a/lib/tasks/event.rake +++ b/lib/tasks/event.rake @@ -50,6 +50,33 @@ namespace :datacite_medra do end end +namespace :datacite_kisti do + desc 'Import kisti dois for all events' + task :import_doi => :environment do + cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i + + Event.update_datacite_kisti(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE']) + end +end + +namespace :datacite_jalc do + desc 'Import jalc dois for all events' + task :import_doi => :environment do + cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i + + Event.update_datacite_jalc(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE']) + end +end + +namespace :datacite_op do + desc 'Import op dois for all events' + task :import_doi => :environment do + cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i + + Event.update_datacite_op(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE']) + end +end + namespace :datacite_orcid_auto_update do desc 'Import orcid ids for all events' task :import_orcid => :environment do