reduce batch size for elasticsearch doi import

datacite · Jan 6, 2021 · 5550d9c · 5550d9c
1 parent 44d08f8
commit 5550d9c
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 17 deletions.
diff --git a/app/models/datacite_doi.rb b/app/models/datacite_doi.rb
@@ -44,7 +44,7 @@ def self.import_by_ids(options = {})
     # TODO remove query for type once STI is enabled
     # SQS message size limit is 256 kB, up to 2 GB with S3
     DataciteDoi.where(type: "DataciteDoi").where(id: from_id..until_id).
-      find_in_batches(batch_size: 200) do |dois|
+      find_in_batches(batch_size: 100) do |dois|
       ids = dois.pluck(:id)
       DataciteDoiImportInBulkJob.perform_later(ids, index: index)
       count += ids.length
@@ -71,7 +71,7 @@ def self.import_by_client(client_id)
 
     # TODO remove query for type once STI is enabled
     DataciteDoi.where(type: "DataciteDoi").where(datacentre: client.id).
-      find_in_batches(batch_size: 200) do |dois|
+      find_in_batches(batch_size: 100) do |dois|
       ids = dois.pluck(:id)
       DataciteDoiImportInBulkJob.perform_later(ids, index: self.active_index)
     end
@@ -87,7 +87,6 @@ def self.import_in_bulk(ids, options = {})
         inactive_index
       end
     errors = 0
-    count = 0
 
     # get database records from array of database ids
     dois = DataciteDoi.where(id: ids)
@@ -115,19 +114,17 @@ def self.import_in_bulk(ids, options = {})
       Rails.logger.error "[Elasticsearch] " + item.inspect
     end
 
-    count += dois.length
-
     if errors > 1
       Rails.logger.error "[Elasticsearch] #{errors} errors importing #{
-                           count
+                          dois.length
                          } DataCite DOIs."
-    elsif count > 0
-      Rails.logger.debug "[Elasticsearch] Imported #{
-                          count
+    elsif dois.length > 0
+      Rails.logger.info "[Elasticsearch] Imported #{
+                         dois.length
                         } DataCite DOIs."
     end
 
-    count
+    dois.length
   rescue Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge,
     Aws::SQS::Errors::RequestEntityTooLarge,
     Faraday::ConnectionFailed,

diff --git a/app/models/other_doi.rb b/app/models/other_doi.rb
@@ -95,7 +95,7 @@ def self.import_by_ids(options = {})
 
     # TODO remove query for type once STI is enabled
     DataciteDoi.where(type: "OtherDoi").where(id: from_id..until_id).
-      find_in_batches(batch_size: 200) do |dois|
+      find_in_batches(batch_size: 100) do |dois|
       ids = dois.pluck(:id)
       OtherDoiImportInBulkJob.perform_later(ids, index: index)
       count += ids.length
@@ -115,7 +115,6 @@ def self.import_in_bulk(ids, options = {})
         inactive_index
       end
     errors = 0
-    count = 0
 
     # get database records from array of database ids
     dois = OtherDoi.where(id: ids)
@@ -143,15 +142,13 @@ def self.import_in_bulk(ids, options = {})
       Rails.logger.error "[Elasticsearch] " + item.inspect
     end
 
-    count += dois.length
-
     if errors > 1
       Rails.logger.error "[Elasticsearch] #{errors} errors importing #{
-                           count
+                           dois.length
                          } Other DOIs."
-    elsif count > 0
+    elsif dois.length > 0
       Rails.logger.debug "[Elasticsearch] Imported #{
-                          count
+                          dois.length
                         } Other DOIs."
     end