support dois from other registration agencies in crossref index. data…

…cite/bolognese#65

Gemfile.lock

@@ -106,7 +106,7 @@ GEM
       latex-decode (~> 0.0)
     binding_of_caller (0.8.0)
       debug_inspector (>= 0.0.1)
-    bolognese (1.2.5)
+    bolognese (1.2.7)
       activesupport (>= 4.2.5, < 6)
       benchmark_methods (~> 0.7)
       bibtex-ruby (~> 4.1)
@@ -214,7 +214,7 @@ GEM
     factory_bot_rails (4.11.1)
       factory_bot (~> 4.11.1)
       railties (>= 3.0.0)
-    faker (1.9.5)
+    faker (1.9.6)
       i18n (>= 0.7)
     faraday (0.14.0)
       multipart-post (>= 1.2, < 3)
@@ -307,7 +307,7 @@ GEM
       rest-client (~> 2.0.2)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    maremma (4.2.3)
+    maremma (4.2.6)
       activesupport (>= 4.2.5, < 6)
       addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)

app/jobs/url_job.rb

@@ -14,13 +14,13 @@ def perform(doi_id)
       response = Doi.get_doi(doi: doi.doi, agency: doi.agency)
       url = response.body.dig('data', 'values', 0, 'data', 'value')
       if url.present?
-        if (doi.is_registered_or_findable? || %w(europ crossref medra).include?(doi.provider_id)) && doi.minted.blank?
+        if (doi.is_registered_or_findable? || %w(europ crossref medra kisti jalc op).include?(doi.provider_id)) && doi.minted.blank?
           doi.update_attributes(url: url, minted: Time.zone.now)
         else
           doi.update_attributes(url: url)
         end
 
-        doi.update_attributes(aasm_state: "findable") if %w(europ crossref medra).include?(doi.provider_id)
+        doi.update_attributes(aasm_state: "findable") if %w(europ crossref medra kisti jalc op).include?(doi.provider_id)
 
         doi.__elasticsearch__.index_document
 

app/models/ability.rb

@@ -12,7 +12,7 @@ def initialize(user)
     if user.role_id == "staff_admin"
       can :manage, :all
       cannot [:new, :create], Doi do |doi|
-        doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || doi.client.symbol.downcase.start_with?("crossref.") || doi.client.symbol.downcase.start_with?("medra."))
+        doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first))
       end
     elsif user.role_id == "staff_user"
       can :read, :all
@@ -65,7 +65,7 @@ def initialize(user)
 
       can [:read, :destroy, :update, :register_url, :validate, :undo, :get_url, :get_urls, :read_landing_page_results], Doi, :client_id => user.client_id
       can [:new, :create], Doi do |doi| 
-        doi.client.prefixes.where(prefix: doi.prefix).present? || doi.client.symbol.downcase.start_with?("crossref.") || doi.client.symbol.downcase.start_with?("medra.")
+        doi.client.prefixes.where(prefix: doi.prefix).present? || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first)
       end
       can [:read], Doi do |doi|
         doi.findable?

app/models/concerns/indexable.rb

@@ -9,7 +9,7 @@ module Indexable
       IndexJob.perform_later(self)
       if self.class.name == "Doi"
         update_column(:indexed, Time.zone.now)
-        send_import_message(self.to_jsonapi) if aasm_state == "findable" unless (Rails.env.test? || client_id == "crossref.citations" || client_id == "medra.citations")
+        send_import_message(self.to_jsonapi) if aasm_state == "findable" unless (Rails.env.test? || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first))
       end
     end
 
@@ -183,12 +183,12 @@ def query(query, options={})
         must << { terms: { "software.raw" => options[:software].split(",") }} if options[:software].present?
         must << { term: { repository_id: options[:repository_id].gsub("/", '\/') }} if options[:repository_id].present?
         must_not << { exists: { field: "deleted_at" }} unless options[:include_deleted]
-        must_not << { terms: { provider_id: ["crossref", "medra"] }} if options[:exclude_registration_agencies]
+        must_not << { terms: { provider_id: ["crossref", "medra", "op"] }} if options[:exclude_registration_agencies]
       elsif self.name == "Doi"
         must << { terms: { aasm_state: options[:state].to_s.split(",") }} if options[:state].present?
         must << { range: { registered: { gte: "#{options[:registered].split(",").min}||/y", lte: "#{options[:registered].split(",").max}||/y", format: "yyyy" }}} if options[:registered].present?
         must << { term: { "client.repository_id" => options[:repository_id].upcase.gsub("/", '\/') }} if options[:repository_id].present?
-        must_not << { terms: { provider_id: ["crossref", "medra"] }} if options[:exclude_registration_agencies]
+        must_not << { terms: { provider_id: ["crossref", "medra", "op"] }} if options[:exclude_registration_agencies]
       elsif self.name == "Event"
         must << { term: { subj_id: options[:subj_id] }} if options[:subj_id].present?
         must << { term: { obj_id: options[:obj_id] }} if options[:obj_id].present?

app/models/doi.rb

@@ -580,7 +580,7 @@ def registerable?
   # end
 
   def is_registered_or_findable?
-    %w(registered findable).include?(aasm_state) || %w(crossref medra).include?(provider_id)
+    %w(registered findable).include?(aasm_state) || %w(crossref medra op).include?(provider_id)
   end
 
   def validatable?

app/models/event.rb

@@ -306,47 +306,55 @@ def self.update_crossref(options={})
   end
 
   def self.update_datacite_crossref(options={})
-    logger = Logger.new(STDOUT)
+    update_datacite_ra(options.merge(ra: "crossref"))
+  end
 
-    size = (options[:size] || 1000).to_i
-    cursor = (options[:cursor] || 0).to_i
+  def self.update_datacite_medra(options={})
+    update_datacite_ra(options.merge(ra: "medra"))
+  end
 
-    response = Event.query(nil, source_id: "datacite-crossref", page: { size: 1, cursor: cursor })
-    logger.info "[Update] #{response.results.total} events for source datacite-crossref."
+  def self.update_datacite_kisti(options={})
+    update_datacite_ra(options.merge(ra: "kisti"))
+  end
 
-    # walk through results using cursor
-    if response.results.total > 0
-      while response.results.results.length > 0 do
-        response = Event.query(nil, source_id: "datacite-crossref", page: { size: size, cursor: cursor })
-        break unless response.results.results.length > 0
+  def self.update_datacite_jalc(options={})
+    update_datacite_ra(options.merge(ra: "jalc"))
+  end
 
-        logger.info "[Update] Updating #{response.results.results.length} datacite-crossref events starting with _id #{cursor + 1}."
-        cursor = response.results.to_a.last[:sort].first.to_i
+  def self.update_datacite_op(options={})
+    update_datacite_ra(options.merge(ra: "op"))
+  end
 
-        dois = response.results.results.map(&:obj_id).uniq
-        CrossrefDoiJob.perform_later(dois, options)
-      end
-    end
+  def self.update_datacite_medra(options={})
+    update_datacite_ra(options.merge(ra: "medra"))
+  end
 
-    response.results.total
+  def self.update_datacite_medra(options={})
+    update_datacite_ra(options.merge(ra: "medra"))
   end
 
   def self.update_datacite_medra(options={})
+    update_datacite_ra(options.merge(ra: "medra"))
+  end
+
+  def self.update_datacite_ra(options={})
     logger = Logger.new(STDOUT)
 
     size = (options[:size] || 1000).to_i
     cursor = (options[:cursor] || 0).to_i
+    ra = options[:ra] || "crossref"
+    source_id = "datacite-#{ra}"
 
-    response = Event.query(nil, source_id: "datacite-medra", page: { size: 1, cursor: cursor })
-    logger.info "[Update] #{response.results.total} events for source datacite-medra."
+    response = Event.query(nil, source_id: source_id, page: { size: 1, cursor: cursor })
+    logger.info "[Update] #{response.results.total} events for source #{source_id}."
 
     # walk through results using cursor
     if response.results.total > 0
       while response.results.results.length > 0 do
-        response = Event.query(nil, source_id: "datacite-medra", page: { size: size, cursor: cursor })
+        response = Event.query(nil, source_id: source_id, page: { size: size, cursor: cursor })
         break unless response.results.results.length > 0
 
-        logger.info "[Update] Updating #{response.results.results.length} datacite-medra events starting with _id #{cursor + 1}."
+        logger.info "[Update] Updating #{response.results.results.length} #{source_id} events starting with _id #{cursor + 1}."
         cursor = response.results.to_a.last[:sort].first.to_i
 
         dois = response.results.results.map(&:obj_id).uniq

lib/tasks/event.rake

@@ -50,6 +50,33 @@ namespace :datacite_medra do
   end
 end
 
+namespace :datacite_kisti do
+  desc 'Import kisti dois for all events'
+  task :import_doi => :environment do
+    cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i
+
+    Event.update_datacite_kisti(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE'])
+  end
+end
+
+namespace :datacite_jalc do
+  desc 'Import jalc dois for all events'
+  task :import_doi => :environment do
+    cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i
+
+    Event.update_datacite_jalc(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE'])
+  end
+end
+
+namespace :datacite_op do
+  desc 'Import op dois for all events'
+  task :import_doi => :environment do
+    cursor = (ENV['CURSOR'] || Event.minimum(:id)).to_i
+
+    Event.update_datacite_op(cursor: cursor, refresh: ENV['REFRESH'], size: ENV['SIZE'])
+  end
+end
+
 namespace :datacite_orcid_auto_update do
   desc 'Import orcid ids for all events'
   task :import_orcid => :environment do