Skip to content

Commit

Permalink
rescue elasticsearch bulk update errors
Browse files Browse the repository at this point in the history
  • Loading branch information
Martin Fenner committed Oct 17, 2018
1 parent 8b47408 commit 91ca365
Showing 1 changed file with 14 additions and 2 deletions.
16 changes: 14 additions & 2 deletions app/models/doi.rb
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def self.index_by_day(options={})

logger = Logger.new(STDOUT)

Doi.where("created >= ?", from_date.strftime("%F") + " 00:00:00").where("created < ?", until_date.strftime("%F") + " 00:00:00").where("updated > indexed").find_in_batches(batch_size: 10) do |dois|
Doi.where("created >= ?", from_date.strftime("%F") + " 00:00:00").where("created < ?", until_date.strftime("%F") + " 00:00:00").where("updated > indexed").find_in_batches(batch_size: 100) do |dois|
response = Doi.__elasticsearch__.client.bulk \
index: Doi.index_name,
type: Doi.document_type,
Expand All @@ -260,12 +260,24 @@ def self.index_by_day(options={})
count += dois.length
dois.each { |doi| doi.update_column(:indexed, Time.zone.now) }
end

if errors > 1
logger.info "[Elasticsearch] #{errors} errors indexing #{count} DOIs created on #{from_date.strftime("%F")}."
elsif count > 1
logger.info "[Elasticsearch] Indexed #{count} DOIs created on #{from_date.strftime("%F")}."
end
rescue Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge => error
logger.info "[Elasticsearch] Error #{error.message} indexing DOIs created on #{from_date.strftime("%F")}."

count = 0

Doi.where("created >= ?", from_date.strftime("%F") + " 00:00:00").where("created < ?", until_date.strftime("%F") + " 00:00:00").where("updated > indexed").find_each do |doi|
IndexJob.perform_later(doi)
doi.update_column(:indexed, Time.zone.now)
count += 1
end

logger.info "[Elasticsearch] Indexed #{count} DOIs created on #{from_date.strftime("%F")}."
end

def uid
Expand Down

0 comments on commit 91ca365

Please sign in to comment.