From c0b84227de0420c3b7ddeb881fc2a7b42f0fa201 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Fri, 11 Jun 2021 03:01:44 +0200 Subject: [PATCH] fix syntax of rake tasks --- app/models/concerns/indexable.rb | 218 +++++++++++++++++++++++-------- app/models/user.rb | 3 +- lib/tasks/claim.rake | 35 ++--- lib/tasks/user.rake | 21 +-- 4 files changed, 200 insertions(+), 77 deletions(-) diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index 26667b7e..665fe05c 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -210,8 +210,7 @@ def count # is inactive. All index configuration changes and bulk importing from the database # happen in the inactive index. # - # For initial setup run "start_aliases" to preserve existing index, or - # "create_index" to start from scratch. + # For initial setup run "create_index" to start from scratch. # # Run "upgrade_index" whenever there are changes in the mappings or settings. # Follow this by "import" to fill the new index, the usen "switch_index" to @@ -220,28 +219,6 @@ def count # TODO: automatically switch aliases when "import" is done. Not easy, as "import" # runs as background jobs. - # convert existing index to alias. Has to be done only once - def start_aliases - alias_name = index_name - index_name = self.index_name + "_v1" - alternate_index_name = self.index_name + "_v2" - - client = Elasticsearch::Model.client - - if client.indices.exists_alias?(name: alias_name) - return "Index #{alias_name} is already an alias." - end - - __elasticsearch__.create_index!(index: index_name) unless __elasticsearch__.index_exists?(index: index_name) - __elasticsearch__.create_index!(index: alternate_index_name) unless __elasticsearch__.index_exists?(index: alternate_index_name) - - # copy old index to first of the new indexes, delete the old index, and alias the old index - client.reindex(body: { source: { index: alias_name }, dest: { index: index_name } }, timeout: "10m", wait_for_completion: false) - - "Created indexes #{index_name} (active) and #{alternate_index_name}." - "Started reindexing in #{index_name}." - end - # track reindexing via the tasks API def monitor_reindex client = Elasticsearch::Model.client @@ -249,53 +226,190 @@ def monitor_reindex tasks.fetch("nodes", {}).inspect end - # convert existing index to alias. Has to be done only once - def finish_aliases - alias_name = index_name - index_name = self.index_name + "_v1" + # create alias + def create_alias(options = {}) + alias_name = options[:alias] || index_name + index_name = options[:index] || self.index_name + "_v1" + # alternate_index_name = options[:index] || self.index_name + "_v2" client = Elasticsearch::Model.client - if client.indices.exists_alias?(name: alias_name) - return "Index #{alias_name} is already an alias." + # indexes in DOI model are aliased from DataciteDoi and OtherDoi models + # TODO switch to DataciteDoi index + # if self.name == "Doi" + # datacite_index_name = DataciteDoi.index_name + "_v1" + # datacite_alternate_index_name = DataciteDoi.index_name + "_v2" + # other_index_name = OtherDoi.index_name + "_v1" + # other_alternate_index_name = OtherDoi.index_name + "_v2" + + # if client.indices.exists_alias?(name: alias_name, index: [datacite_index_name]) + # "Alias #{alias_name} for index #{datacite_index_name} already exists." + # else + # client.indices.put_alias index: datacite_index_name, name: alias_name + # "Created alias #{alias_name} for index #{datacite_index_name}." + # end + # if client.indices.exists_alias?(name: alias_name, index: [other_index_name]) + # "Alias #{alias_name} for index #{other_index_name} already exists." + # else + # client.indices.put_alias index: other_index_name, name: alias_name + # "Created alias #{alias_name} for index #{other_index_name}." + # end + # else + if client.indices.exists_alias?(name: alias_name, index: [index_name]) + "Alias #{alias_name} for index #{index_name} already exists." + else + # alias index is writeable unless it is for OtherDoi index + client.indices.update_aliases( + body: { + actions: [ + { + add: { + index: index_name, + alias: alias_name, + is_write_index: name != "OtherDoi", + }, + }, + ], + }, + ) + + "Created alias #{alias_name} for index #{index_name}." end + # end + end + + # list all aliases + def list_aliases + client = Elasticsearch::Model.client + cat_client = Elasticsearch::API::Cat::CatClient.new(client) + puts cat_client.aliases(s: "alias") + end + + # delete alias + def delete_alias(options = {}) + alias_name = options[:alias] || index_name + index_name = (options[:index] || self.index_name) + "_v1" + alternate_index_name = (options[:index] || self.index_name) + "_v2" - __elasticsearch__.delete_index!(index: alias_name) if __elasticsearch__.index_exists?(index: alias_name) - client.indices.put_alias index: index_name, name: alias_name + client = Elasticsearch::Model.client - "Converted index #{alias_name} into an alias." + # indexes in DOI model are aliased from DataciteDoi and OtherDoi models + # TODO switch to DataciteDoi index + # if self.name == "Doi" + # datacite_index_name = DataciteDoi.index_name + "_v1" + # datacite_alternate_index_name = DataciteDoi.index_name + "_v2" + # other_index_name = OtherDoi.index_name + "_v1" + # other_alternate_index_name = OtherDoi.index_name + "_v2" + + # if client.indices.exists_alias?(name: alias_name, index: [datacite_index_name]) + # client.indices.delete_alias index: datacite_index_name, name: alias_name + # "Deleted alias #{alias_name} for index #{datacite_index_name}." + # end + # if client.indices.exists_alias?(name: alias_name, index: [datacite_alternate_index_name]) + # client.indices.delete_alias index: datacite_alternate_index_name, name: alias_name + # "Deleted alias #{alias_name} for index #{datacite_alternate_index_name}." + # end + # if client.indices.exists_alias?(name: alias_name, index: [other_index_name]) + # client.indices.delete_alias index: other_index_name, name: alias_name + # "Deleted alias #{alias_name} for index #{other_index_name}." + # end + # if client.indices.exists_alias?(name: alias_name, index: [other_alternate_index_name]) + # client.indices.delete_alias index: other_alternate_index_name, name: alias_name + # "Deleted alias #{alias_name} for index #{other_alternate_index_name}." + # end + # else + if client.indices.exists_alias?(name: alias_name, index: [index_name]) + client.indices.delete_alias index: index_name, name: alias_name + "Deleted alias #{alias_name} for index #{index_name}." + end + if client.indices.exists_alias?( + name: alias_name, index: [alternate_index_name], + ) + client.indices.delete_alias index: alternate_index_name, + name: alias_name + "Deleted alias #{alias_name} for index #{alternate_index_name}." + end + # end end # create both indexes used for aliasing - def create_index - alias_name = index_name - index_name = self.index_name + "_v1" - alternate_index_name = self.index_name + "_v2" + def create_index(options = {}) + alias_name = options[:alias] || index_name + index_name = (options[:index] || self.index_name) + "_v1" + alternate_index_name = (options[:index] || self.index_name) + "_v2" + client = Elasticsearch::Model.client - __elasticsearch__.create_index!(index: index_name) unless __elasticsearch__.index_exists?(index: index_name) - __elasticsearch__.create_index!(index: alternate_index_name) unless __elasticsearch__.index_exists?(index: alternate_index_name) + # delete index if it has the same name as the alias + if __elasticsearch__.index_exists?(index: alias_name) && + !client.indices.exists_alias?(name: alias_name) + __elasticsearch__.delete_index!(index: alias_name) + end - # index_name is the active index - client = Elasticsearch::Model.client - client.indices.put_alias index: index_name, name: alias_name unless client.indices.exists_alias?(name: alias_name) + create_template if name == "DataciteDoi" || name == "OtherDoi" + + # indexes in DOI model are aliased from DataciteDoi and OtherDoi models + # TODO switch to DataciteDoi index + # if self.name == "Doi" + # datacite_index_name = DataciteDoi.index_name + "_v1" + # datacite_alternate_index_name = DataciteDoi.index_name + "_v2" + # other_index_name = OtherDoi.index_name + "_v1" + # other_alternate_index_name = OtherDoi.index_name + "_v2" + + # self.__elasticsearch__.create_index!(index: datacite_index_name) unless self.__elasticsearch__.index_exists?(index: datacite_index_name) + # self.__elasticsearch__.create_index!(index: datacite_alternate_index_name) unless self.__elasticsearch__.index_exists?(index: datacite_alternate_index_name) + # self.__elasticsearch__.create_index!(index: other_index_name) unless self.__elasticsearch__.index_exists?(index: other_index_name) + # self.__elasticsearch__.create_index!(index: other_alternate_index_name) unless self.__elasticsearch__.index_exists?(index: other_alternate_index_name) + + # "Created indexes #{datacite_index_name}, #{other_index_name}, #{datacite_alternate_index_name}, and #{other_alternate_index_name}." + # else + unless __elasticsearch__.index_exists?(index: index_name) + __elasticsearch__.create_index!(index: index_name) + end + unless __elasticsearch__.index_exists?(index: alternate_index_name) + __elasticsearch__.create_index!(index: alternate_index_name) + end - "Created indexes #{index_name} (active) and #{alternate_index_name}." + "Created indexes #{index_name} and #{alternate_index_name}." + # end end - # delete both indexes used for aliasing - def delete_index - alias_name = index_name + # delete index and both indexes used for aliasing + def delete_index(options = {}) + # client = Elasticsearch::Model.client + + if options[:index] + __elasticsearch__.delete_index!(index: options[:index]) + return "Deleted index #{options[:index]}." + end + + # alias_name = index_name index_name = self.index_name + "_v1" alternate_index_name = self.index_name + "_v2" - client = Elasticsearch::Model.client - client.indices.delete_alias index: index_name, name: alias_name if client.indices.exists_alias?(name: alias_name, index: [index_name]) - client.indices.delete_alias index: alternate_index_name, name: alias_name if client.indices.exists_alias?(name: alias_name, index: [alternate_index_name]) - - __elasticsearch__.delete_index!(index: index_name) if __elasticsearch__.index_exists?(index: index_name) - __elasticsearch__.delete_index!(index: alternate_index_name) if __elasticsearch__.index_exists?(index: alternate_index_name) + # indexes in DOI model are aliased from DataciteDoi and OtherDoi models + # TODO switch to DataciteDoi index + # if self.name == "Doi" + # datacite_index_name = DataciteDoi.index_name + "_v1" + # datacite_alternate_index_name = DataciteDoi.index_name + "_v2" + # other_index_name = OtherDoi.index_name + "_v1" + # other_alternate_index_name = OtherDoi.index_name + "_v2" + + # self.__elasticsearch__.delete_index!(index: datacite_index_name) if self.__elasticsearch__.index_exists?(index: datacite_index_name) + # self.__elasticsearch__.delete_index!(index: datacite_alternate_index_name) if self.__elasticsearch__.index_exists?(index: datacite_alternate_index_name) + # self.__elasticsearch__.delete_index!(index: other_index_name) if self.__elasticsearch__.index_exists?(index: other_index_name) + # self.__elasticsearch__.delete_index!(index: other_alternate_index_name) if self.__elasticsearch__.index_exists?(index: other_alternate_index_name) + + # "Deleted indexes #{datacite_index_name}, #{other_index_name}, #{datacite_alternate_index_name}, and #{other_alternate_index_name}." + # else + if __elasticsearch__.index_exists?(index: index_name) + __elasticsearch__.delete_index!(index: index_name) + end + if __elasticsearch__.index_exists?(index: alternate_index_name) + __elasticsearch__.delete_index!(index: alternate_index_name) + end "Deleted indexes #{index_name} and #{alternate_index_name}." + # end end # delete and create inactive index to use current mappings diff --git a/app/models/user.rb b/app/models/user.rb index 86d37851..857bef84 100755 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -193,7 +193,6 @@ def queue_user_job end def self.delete_expired_token(index: nil) - size = (options[:size] || 1000).to_i query = "orcid_expires_at:[1970-01-02 TO #{Date.today.strftime('%F')}]" response = User.query(query, index: index, page: { size: 1, cursor: [] }) @@ -204,7 +203,7 @@ def self.delete_expired_token(index: nil) cursor = [] while !response.results.results.empty? - response = User.query(query, index: index, page: { size: size, cursor: cursor }) + response = User.query(query, index: index, page: { size: 1000, cursor: cursor }) break if response.results.results.empty? Rails.logger.info "Deleting #{response.results.length} User ORCID tokens starting with _id #{response.results.to_a.first[:_id]}." diff --git a/lib/tasks/claim.rake b/lib/tasks/claim.rake index e8ead3c1..2f2f87b8 100644 --- a/lib/tasks/claim.rake +++ b/lib/tasks/claim.rake @@ -31,19 +31,24 @@ namespace :claim do puts Claim.active_index + " is the active index." end - desc "Start using alias indexes for claims" - task start_aliases: :environment do - puts Claim.start_aliases - end - desc "Monitor reindexing for claims" task monitor_reindex: :environment do puts Claim.monitor_reindex end - desc "Wrap up starting using alias indexes for claims" - task finish_aliases: :environment do - puts Claim.finish_aliases + desc "Create alias for claims" + task create_alias: :environment do + puts Claim.create_alias(index: ENV["INDEX"], alias: ENV["ALIAS"]) + end + + desc "List aliases for claims" + task list_aliases: :environment do + puts Claim.list_aliases + end + + desc "Delete alias for claims" + task delete_alias: :environment do + puts Claim.delete_alias(index: ENV["INDEX"], alias: ENV["ALIAS"]) end desc "Import all claims" @@ -81,13 +86,13 @@ namespace :claim do end end - desc "Push all waiting claims" - task waiting: :environment do - Claim.waiting.find_each do |claim| - ClaimJob.perform_later(claim) - puts "[#{claim.aasm_state}] Pushed waiting claim #{claim.doi} for user #{claim.orcid} to ORCID." - end - end + # desc "Push all waiting claims" + # task waiting: :environment do + # Claim.waiting.find_each do |claim| + # ClaimJob.perform_later(claim) + # puts "[#{claim.aasm_state}] Pushed waiting claim #{claim.doi} for user #{claim.orcid} to ORCID." + # end + # end desc "Get notification_access_token" task get_notification_access_token: :environment do diff --git a/lib/tasks/user.rake b/lib/tasks/user.rake index cccc8a04..f2b70653 100644 --- a/lib/tasks/user.rake +++ b/lib/tasks/user.rake @@ -31,19 +31,24 @@ namespace :user do puts User.active_index + " is the active index." end - desc "Start using alias indexes for users" - task start_aliases: :environment do - puts User.start_aliases - end - desc "Monitor reindexing for users" task monitor_reindex: :environment do puts User.monitor_reindex end - desc "Wrap up starting using alias indexes for users" - task finish_aliases: :environment do - puts User.finish_aliases + desc "Create alias for users" + task create_alias: :environment do + puts User.create_alias(index: ENV["INDEX"], alias: ENV["ALIAS"]) + end + + desc "List aliases for users" + task list_aliases: :environment do + puts User.list_aliases + end + + desc "Delete alias for users" + task delete_alias: :environment do + puts User.delete_alias(index: ENV["INDEX"], alias: ENV["ALIAS"]) end desc "Import all users"