From 4e88225c67d776eac2ee779bf3fbc1c54db080d3 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Wed, 26 Dec 2018 17:38:06 +0100 Subject: [PATCH] support indexing by client --- app/models/doi.rb | 11 ++++++++--- lib/tasks/doi.rake | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/models/doi.rb b/app/models/doi.rb index 8d3957124..6ec3b0b20 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -443,10 +443,11 @@ def self.index(options={}) from_date = options[:from_date].present? ? Date.parse(options[:from_date]) : Date.current until_date = options[:until_date].present? ? Date.parse(options[:until_date]) : Date.current index_time = options[:index_time].presence || Time.zone.now.utc.iso8601 - + client_id = options[:client_id] + # get every day between from_date and until_date (from_date..until_date).each do |d| - DoiIndexByDayJob.perform_later(from_date: d.strftime("%F"), index_time: index_time) + DoiIndexByDayJob.perform_later(from_date: d.strftime("%F"), index_time: index_time, client_id: client_id) puts "Queued indexing for DOIs created on #{d.strftime("%F")}." end end @@ -455,13 +456,17 @@ def self.index_by_day(options={}) return nil unless options[:from_date].present? from_date = Date.parse(options[:from_date]) index_time = options[:index_time].presence || Time.zone.now.utc.iso8601 + client_id = options[:client_id] errors = 0 count = 0 logger = Logger.new(STDOUT) - Doi.where(created: from_date.midnight..from_date.end_of_day).where("indexed < ?", index_time).find_in_batches(batch_size: 250) do |dois| + collection = Doi.where(created: from_date.midnight..from_date.end_of_day).where("indexed < ?", index_time) + collection = collection.where(client_id: client_id) if client_id.present? + + collection.find_in_batches(batch_size: 250) do |dois| response = Doi.__elasticsearch__.client.bulk \ index: Doi.index_name, type: Doi.document_type, diff --git a/lib/tasks/doi.rake b/lib/tasks/doi.rake index c97a8e520..fb8ee4ac0 100644 --- a/lib/tasks/doi.rake +++ b/lib/tasks/doi.rake @@ -61,8 +61,9 @@ namespace :doi do end index_time = ENV['INDEX_TIME'] || Time.zone.now.utc.iso8601 + client_id = ENV['CLIENT_ID'] - Doi.index(from_date: from_date, until_date: until_date, index_time: index_time) + Doi.index(from_date: from_date, until_date: until_date, index_time: index_time, client_id: client_id) end desc 'Index DOIs per day'