Merge pull request #587 from datacite/loop-through-dois-background

Run loop_through_dois in background

app/jobs/loop_through_dois_job.rb

@@ -0,0 +1,10 @@
+class LoopThroughDoisJob < ActiveJob::Base
+  queue_as :lupo_background
+
+  def perform(ids, options={})
+    ids.each do |id|
+      Object.const_get(options[:job_name]).perform_later(id, options)
+      sleep 0.1
+    end
+  end
+end

app/models/doi.rb

@@ -1873,21 +1873,20 @@ def self.transfer(options={})
     response.results.total
   end
 
-  # Transverses the index in batches and using the cursor pagination and executes a Job that matches the query and filer
+  # Transverses the index in batches and using the cursor pagination and executes a Job that matches the query and filter
   # Options:
   # +filter+:: paramaters to filter the index
   # +label+:: String to output in the logs printout
   # +query+:: ES query to filter the index
   # +job_name+:: Acive Job class name of the Job that would be executed on every matched results
-  def self.loop_through_dois(options)
+  def self.loop_through_dois(options={})
     size = (options[:size] || 1000).to_i
-    cursor = [options[:from_id] || Doi.minimum(:id).to_i, options[:until_id] || Doi.maximum(:id).to_i]
+    cursor = []
     filter = options[:filter] || {}
     label = options[:label] || ""
-    job_name = options[:job_name] || ""
+    options[:job_name] ||= ""
     query = options[:query] || nil
 
-
     response = Doi.query(query, filter.merge(page: { size: 1, cursor: [] }))
     Rails.logger.info "#{label} #{response.results.total} Dois with #{label}."
 
@@ -1902,15 +1901,11 @@ def self.loop_through_dois(options)
         Rails.logger.info "#{label} Cursor: #{cursor} "
 
         ids = response.results.results.map(&:uid)
-        ids.each do |id|
-          Object.const_get(job_name).perform_later(id, options)
-          sleep 0.1
-        end
+        LoopThroughDoisJob.perform_later(ids, options)
       end
     end
   end
 
-
   # save to metadata table when xml has changed
   def save_metadata
     metadata.build(doi: self, xml: xml, namespace: schema_version) if xml.present? && xml_changed?

lib/tasks/doi.rake

@@ -92,8 +92,6 @@ namespace :doi do
   desc "Set schema version"
   task set_schema_version: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "+aasm_state:(findable OR registered) -schema_version:*",
       label: "[SetSchemaVersion]",
       job_name: "SchemaVersionJob",
@@ -104,8 +102,6 @@ namespace :doi do
   desc "Set registration agency"
   task set_registration_agency: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "agency:DataCite OR agency:Crossref",
       label: "[SetRegistrationAgency]",
       job_name: "UpdateDoiJob",
@@ -116,8 +112,6 @@ namespace :doi do
   desc "Set license"
   task set_license: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "rights_list:* AND !rights_list.rightsIdentifier",
       label: "[SetLicense]",
       job_name: "UpdateDoiJob",
@@ -128,8 +122,6 @@ namespace :doi do
   desc "Set language"
   task set_language: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "language:*",
       label: "[SetLanguage]",
       job_name: "UpdateDoiJob",
@@ -140,8 +132,6 @@ namespace :doi do
   desc "Set identifiers"
   task set_identifiers: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "identifiers.identifierType:DOI",
       label: "[SetIdentifiers]",
       job_name: "UpdateDoiJob",
@@ -152,8 +142,6 @@ namespace :doi do
   desc "Set field of science"
   task set_field_of_science: :environment do
     options = {
-      from_id: (ENV["FROM_ID"] || Doi.minimum(:id)).to_i,
-      until_id: (ENV["UNTIL_ID"] || Doi.maximum(:id)).to_i,
       query: "subjects.subjectScheme:FOR",
       label: "[SetFieldOfScience]",
       job_name: "UpdateDoiJob",