From b61a63b20c4c9c0c1bcd9c7020cecd8c4ab24ff0 Mon Sep 17 00:00:00 2001
From: Martin Fenner <martin.fenner@datacite.org>
Date: Tue, 19 Feb 2019 22:11:37 +0100
Subject: [PATCH] added elasticsearch tests. #205

---
 app/controllers/concerns/facetable.rb         |   8 +-
 app/controllers/prefixes_controller.rb        |   2 +-
 .../provider_prefixes_controller.rb           |   2 +-
 app/jobs/transfer_job.rb                      |   6 +-
 app/models/client.rb                          |   6 +-
 app/models/client_prefix.rb                   |   2 +-
 app/models/concerns/cacheable.rb              | 111 +----------
 app/models/concerns/indexable.rb              |   1 +
 app/models/doi.rb                             | 108 ++++++++---
 app/models/provider_prefix.rb                 |   2 +-
 app/models/user.rb                            |   4 +-
 lib/tasks/client.rake                         |   6 +-
 spec/concerns/facetable_spec.rb               |  68 +++----
 spec/concerns/indexable_spec.rb               | 183 +++++++++---------
 spec/models/client_spec.rb                    |  17 --
 spec/models/doi_spec.rb                       |  88 +++++++++
 spec/requests/clients_spec.rb                 |  66 ++++---
 spec/requests/dois_spec.rb                    |  12 +-
 spec/requests/providers_spec.rb               |  33 ++--
 spec/requests/works_spec.rb                   |   8 +-
 spec/support/elasticsearch_helper.rb          |   2 +-
 21 files changed, 369 insertions(+), 366 deletions(-)

diff --git a/app/controllers/concerns/facetable.rb b/app/controllers/concerns/facetable.rb
index 6ee2aad05..befe5f2b6 100644
--- a/app/controllers/concerns/facetable.rb
+++ b/app/controllers/concerns/facetable.rb
@@ -8,12 +8,6 @@ module Facetable
       "apac" => "Asia Pacific",
       "emea" => "EMEA" }
 
-    def client_year_facet(params, collection)
-      [{ id: params[:year],
-         title: params[:year],
-         count: collection.where('YEAR(datacentre.created) = ?', params[:year]).count }]
-    end
-
     def facet_by_year(arr)
       arr.map do |hsh|
         { "id" => hsh["key_as_string"][0..3],
@@ -94,7 +88,7 @@ def facet_by_provider(arr)
       # generate hash with id and name for each provider in facet
 
       ids = arr.map { |hsh| hsh["key"] }.join(",")
-      providers = Provider.find_by_ids(ids).results.reduce({}) do |sum, p|
+      providers = Provider.find_by_ids(ids, size: 1000).results.reduce({}) do |sum, p|
         sum[p.symbol.downcase] = p.name
         sum
       end
diff --git a/app/controllers/prefixes_controller.rb b/app/controllers/prefixes_controller.rb
index b175b8151..57a662cc8 100644
--- a/app/controllers/prefixes_controller.rb
+++ b/app/controllers/prefixes_controller.rb
@@ -43,7 +43,7 @@ def index
       providers = providers
                   .sort { |a, b| b[1] <=> a[1] }
                   .reduce([]) do |sum, i|
-                                if provider = cached_providers.find { |m| m.id == i[0] }
+                                if provider = Provider.where(symbol: i[0]).first
                                   sum << { id: provider.symbol.downcase, title: provider.name, count: i[1] }
                                 end
 
diff --git a/app/controllers/provider_prefixes_controller.rb b/app/controllers/provider_prefixes_controller.rb
index 233543c3b..7b5bdcd3e 100644
--- a/app/controllers/provider_prefixes_controller.rb
+++ b/app/controllers/provider_prefixes_controller.rb
@@ -44,7 +44,7 @@ def index
       providers = providers
                   .sort { |a, b| b[1] <=> a[1] }
                   .reduce([]) do |sum, i|
-                                if provider = cached_providers.find { |m| m.id == i[0] }
+                                if provider = Provider.where(symbol: i[0]).first
                                   sum << { id: provider.symbol.downcase, title: provider.name, count: i[1] }
                                 end
 
diff --git a/app/jobs/transfer_job.rb b/app/jobs/transfer_job.rb
index 9da7f6241..d23ae6e15 100644
--- a/app/jobs/transfer_job.rb
+++ b/app/jobs/transfer_job.rb
@@ -15,11 +15,11 @@ def perform(doi_id, options={})
 
       doi.__elasticsearch__.index_document
 
-      logger.info "[MySQL] Transferred DOI #{doi.doi}."
+      logger.info "[Transfer] Transferred DOI #{doi.doi}."
     elsif doi.present?
-      logger.info "[MySQL] Error transferring DOI " + doi_id + ": no target client"
+      logger.info "[Transfer] Error transferring DOI " + doi_id + ": no target client"
     else
-      logger.info "[MySQL] Error transferring DOI " + doi_id + ": not found"
+      logger.info "[Transfer] Error transferring DOI " + doi_id + ": not found"
     end
   end
 end
\ No newline at end of file
diff --git a/app/models/client.rb b/app/models/client.rb
index 9836ea169..631cb6fd6 100644
--- a/app/models/client.rb
+++ b/app/models/client.rb
@@ -150,7 +150,7 @@ def provider_id
   end
 
   def provider_id=(value)
-    r = cached_provider_response(value)
+    r = Provider.where(symbol: value).first
     return nil unless r.present?
 
     write_attribute(:allocator, r.id)
@@ -182,7 +182,7 @@ def target_id=(value)
 
     target = c.records.first
 
-    Doi.transfer(from_date: "2011-01-01", client_id: id, target_id: target.id)
+    Doi.transfer(client_id: symbol.downcase, target_id: target.id)
   end
 
   def index_all_dois
@@ -207,7 +207,7 @@ def password_input=(value)
 
   # backwards compatibility
   def member
-    cached_member_response(provider_id) if provider_id.present?
+    Provider.where(symbol: provider_id).first if provider_id.present?
   end
 
   def year
diff --git a/app/models/client_prefix.rb b/app/models/client_prefix.rb
index 2b5690d81..a50bab96b 100644
--- a/app/models/client_prefix.rb
+++ b/app/models/client_prefix.rb
@@ -34,7 +34,7 @@ def client_id
 
   # workaround for non-standard database column names and association
   def client_id=(value)
-    r = cached_client_response(value)
+    r = ::Client.where(symbol: value).first
     fail ActiveRecord::RecordNotFound unless r.present?
 
     self.datacentre = r.id
diff --git a/app/models/concerns/cacheable.rb b/app/models/concerns/cacheable.rb
index 39046f979..8bc00272c 100644
--- a/app/models/concerns/cacheable.rb
+++ b/app/models/concerns/cacheable.rb
@@ -2,23 +2,6 @@ module Cacheable
   extend ActiveSupport::Concern
 
   included do
-    def cached_doi_count(options={})
-      Rails.cache.fetch("cached_doi_count/#{id}", expires_in: 24.hours, force: options[:force]) do
-        return [] if Rails.env.test?
-
-        if self.class.name == "Provider" && symbol != "ADMIN"
-          collection = Doi.joins(:client).where("datacentre.allocator = ?", id)
-        elsif self.class.name == "Client"
-          collection = Doi.where(datacentre: id)
-        else
-          collection = Doi
-        end
-
-        years = collection.order("YEAR(dataset.created)").group("YEAR(dataset.created)").count
-        years = years.map { |k,v| { id: k, title: k, count: v } }
-      end
-    end
-    
     def cached_metadata_count(options={})
       Rails.cache.fetch("cached_metadata_count/#{id}", expires_in: 6.hours, force: options[:force]) do
         return [] if Rails.env.test?
@@ -50,16 +33,6 @@ def cached_media_count(options={})
       end
     end
 
-    def fetch_cached_metadata_version
-      if updated.present? && Rails.application.config.action_controller.perform_caching
-        Rails.cache.fetch("cached_metadata_version/#{doi}-#{updated.iso8601}") do
-          current_metadata ? current_metadata.metadata_version : 0
-        end
-      else
-        current_metadata ? current_metadata.metadata_version : 0
-      end
-    end
-
     def cached_prefixes_totals params={}
       if Rails.application.config.action_controller.perform_caching
         Rails.cache.fetch("cached_prefixes_totals/#{params}", expires_in: 24.hours) do
@@ -90,19 +63,9 @@ def cached_clients_totals params={}
       end
     end
 
-    def cached_client_response(id, options={})
-      if Rails.application.config.action_controller.perform_caching
-        Rails.cache.fetch("client_response/#{id}", expires_in: 1.day) do
-          Client.where(symbol: id).first
-        end
-      else
-        Client.where(symbol: id).first
-      end
-    end
-
     def cached_prefix_response(prefix, options={})
       if Rails.application.config.action_controller.perform_caching
-        Rails.cache.fetch("prefix_response/#{prefix}", expires_in: 7.days) do
+        Rails.cache.fetch("prefix_response/#{prefix}", expires_in: 24.hours) do
           Prefix.where(prefix: prefix).first
         end
       else
@@ -110,29 +73,6 @@ def cached_prefix_response(prefix, options={})
       end
     end
 
-    def cached_provider_response(id, options={})
-      if Rails.application.config.action_controller.perform_caching
-        Rails.cache.fetch("provider_response/#{id}", expires_in: 1.day) do
-          Provider.where(symbol: id).first
-        end
-      else
-        Provider.where(symbol: id).first
-      end
-    end
-
-    def cached_providers
-      Rails.cache.fetch("providers", expires_in: 1.day) do
-        Provider.all.select(:id, :symbol, :name, :created)
-      end
-    end
-
-    def cached_member_response(id)
-      Rails.cache.fetch("member_response/#{id}", expires_in: 12.hours) do
-        member = Member.where(id: id)
-        member.present? ? member[:data] : nil
-      end
-    end
-
     def cached_resource_type_response(id)
       Rails.cache.fetch("resource_type_response/#{id}", expires_in: 1.month) do
         resource_type = ResourceType.where(id: id)
@@ -152,12 +92,6 @@ def cached_repository_response(id, options={})
   end
 
   module ClassMethods
-    def cached_providers
-      Rails.cache.fetch("providers", expires_in: 1.day) do
-        Provider.all.select(:id, :symbol, :name, :updated)
-      end
-    end
-
     def cached_metadata_count
       Rails.cache.fetch("cached_metadata_count", expires_in: 6.hours) do
         return [] if Rails.env.test?
@@ -175,48 +109,5 @@ def cached_media_count
         years = years.map { |k,v| { id: k, title: k, count: v } }
       end
     end
-
-    # def cached_clients
-    #   Rails.cache.fetch("clients", expires_in: 1.day) do
-    #     Client.all.select(:id, :symbol, :name, :updated)
-    #   end
-    # end
-
-    def cached_providers_response(options={})
-      Rails.cache.fetch("provider_response", expires_in: 1.day) do
-        Providers.where(options)
-      end
-    end
-
-    def cached_provider_response(symbol)
-      Rails.cache.fetch("provider_response/#{symbol}", expires_in: 1.day) do
-        Provider.where(symbol: symbol).first
-      end
-    end
-
-    def cached_datasets_clients_join(options={})
-      Rails.cache.fetch("clients", expires_in: 1.day) do
-        Dataset.joins(:clients).where("client.symbol" => "dataset.allocator")
-      end
-    end
-
-    def cached_clients_response(options={})
-      Rails.cache.fetch("clients_response", expires_in: 1.day) do
-        # collection = cached_datasets_clients_joins
-        collection.each do |line|
-          dc = Client.find(line[:datacentre])
-          line[:client_id] = dc.uid
-          line[:client_name] = dc.name
-        end
-
-        collection.map{|doi| { id: doi[:id],  client_id: doi[:client_id],  name: doi[:client_name] }}.group_by { |d| d[:client_id] }.map{ |k, v| { id: k, title: v.first[:name], count: v.count} }
-      end
-    end
-
-    def cached_client_response(id, options={})
-      Rails.cache.fetch("client_response/#{id}", expires_in: 1.day) do
-        Client.where(symbol: id).first
-      end
-    end
   end
 end
diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb
index e3d9aaa06..a1cce96f9 100644
--- a/app/models/concerns/indexable.rb
+++ b/app/models/concerns/indexable.rb
@@ -101,6 +101,7 @@ def find_by_ids(ids, options={})
 
     def query(query, options={})
       aggregations = query_aggregations
+      options[:page] ||= { size: 25, number: 1 }
 
       # enable cursor-based pagination for DOIs
       if self.name == "Doi" && options.dig(:page, :cursor).present?
diff --git a/app/models/doi.rb b/app/models/doi.rb
index a4eb7b779..d4fdd505f 100644
--- a/app/models/doi.rb
+++ b/app/models/doi.rb
@@ -349,7 +349,7 @@ def self.find_by_id(id, options={})
     __elasticsearch__.search({
       query: {
         term: {
-          doi: id.downcase
+          doi: id.upcase
         }
       },
       aggregations: query_aggregations
@@ -379,9 +379,11 @@ def self.import_one(doi_id: nil)
     # update_attributes will trigger validations and Elasticsearch indexing
     doi.update_attributes(attrs)
     logger.info "[MySQL] Imported metadata for DOI " + doi.doi + "."
+    doi
   rescue TypeError, NoMethodError, RuntimeError, ActiveRecord::StatementInvalid, ActiveRecord::LockWaitTimeout => error
     logger.error "[MySQL] Error importing metadata for " + doi.doi + ": " + error.message
     Bugsnag.notify(error)
+    doi
   end
 
   def self.import_all(options={})
@@ -393,6 +395,8 @@ def self.import_all(options={})
     (from_date..until_date).each do |d|
       DoiImportByDayJob.perform_later(from_date: d.strftime("%F"), client_id: client_id)
     end
+
+    (from_date..until_date).to_a.length
   end
 
   def self.import_missing(options={})
@@ -404,47 +408,64 @@ def self.import_missing(options={})
     (from_date..until_date).each do |d|
       DoiImportByDayMissingJob.perform_later(from_date: d.strftime("%F"), client_id: client_id)
     end
+
+    (from_date..until_date).to_a.length
   end
 
   def self.import_by_day(options={})
     return nil unless options[:from_date].present?
     from_date = Date.parse(options[:from_date])
-    client_id = options[:client_id]
 
     count = 0
 
     logger = Logger.new(STDOUT)
 
-    collection = Doi.where(created: from_date.midnight..from_date.end_of_day)
-    collection = collection.where(datacentre: client_id) if client_id.present?
+    response = Doi.query("created:[#{from_date.strftime("%F")} TO #{from_date.strftime("%F")}]", page: { size: 0, cursor: 1 })
+    logger.info "#{response.results.total} DOIs found created on #{from_date.strftime("%F")}."
 
-    collection.find_each do |doi|
-      begin
-        # ignore broken xml
-        string = doi.current_metadata.present? ? doi.from_xml(doi.current_metadata.xml.to_s.force_encoding("UTF-8")) : nil
-        unless string.present?
-          logger.error "[MySQL] No metadata for DOI " + doi.doi + " found."
-          return nil
+    if response.results.total > 0
+      # walk through results using cursor
+      prev_cursor = 0
+      cursor = 1
+      
+      while cursor > prev_cursor do
+        response = Doi.query("created:[#{from_date.strftime("%F")} TO #{from_date.strftime("%F")}]", page: { size: 1000, cursor: cursor })
+        prev_cursor = cursor
+        cursor = Array.wrap(response.results.results.last.to_h[:sort]).first.to_i
+
+        response.records.each do |doi|
+          begin
+            # ignore broken xml
+            string = doi.current_metadata.present? ? doi.from_xml(doi.current_metadata.xml.to_s.force_encoding("UTF-8")) : nil
+            unless string.present?
+              logger.error "[MySQL] No metadata for DOI " + doi.doi + " found."
+              return nil
+            end
+            
+            meta = doi.read_datacite(string: string, sandbox: doi.sandbox)
+            attrs = %w(creators contributors titles publisher publication_year types descriptions container sizes formats language dates identifiers related_identifiers funding_references geo_locations rights_list subjects content_url).map do |a|
+              [a.to_sym, meta[a]]
+            end.to_h.merge(schema_version: meta["schema_version"] || "http://datacite.org/schema/kernel-4", version_info: meta["version"], xml: string)
+    
+            # update_columns will NOT trigger validations and Elasticsearch indexing
+            doi.update_columns(attrs)
+
+            doi.__elasticsearch__.index_document
+          rescue TypeError, NoMethodError, RuntimeError, ActiveRecord::StatementInvalid, ActiveRecord::LockWaitTimeout => error
+            logger.error "[MySQL] Error importing metadata for " + doi.doi + ": " + error.message
+            Bugsnag.notify(error)
+          else
+            count += 1
+          end
+        end
+    
+        if count > 0
+          logger.info "[MySQL] Imported metadata for #{count} DOIs created on #{options[:from_date]}."
         end
-        
-        meta = doi.read_datacite(string: string, sandbox: doi.sandbox)
-        attrs = %w(creators contributors titles publisher publication_year types descriptions container sizes formats language dates identifiers related_identifiers funding_references geo_locations rights_list subjects content_url).map do |a|
-          [a.to_sym, meta[a]]
-        end.to_h.merge(schema_version: meta["schema_version"] || "http://datacite.org/schema/kernel-4", version_info: meta["version"], xml: string)
-
-        # update_columns will NOT trigger validations and Elasticsearch indexing
-        doi.update_columns(attrs)
-      rescue TypeError, NoMethodError, RuntimeError, ActiveRecord::StatementInvalid, ActiveRecord::LockWaitTimeout => error
-        logger.error "[MySQL] Error importing metadata for " + doi.doi + ": " + error.message
-        Bugsnag.notify(error)
-      else
-        count += 1
       end
     end
 
-    if count > 0
-      logger.info "[MySQL] Imported metadata for #{count} DOIs created on #{options[:from_date]}."
-    end
+    count
   end
 
   def self.import_by_day_missing(options={})
@@ -499,6 +520,8 @@ def self.import_by_day_missing(options={})
         end
       end
     end
+
+    count
   end
 
   def self.index(options={})
@@ -511,6 +534,8 @@ def self.index(options={})
     (from_date..until_date).each do |d|
       DoiIndexByDayJob.perform_later(from_date: d.strftime("%F"), index_time: index_time, client_id: client_id)
     end
+
+    (from_date..until_date).to_a.length
   end
 
   def self.index_by_day(options={})
@@ -548,6 +573,8 @@ def self.index_by_day(options={})
     elsif count > 1
       logger.info "[Elasticsearch] Indexed #{count} DOIs created on #{options[:from_date]}."
     end
+
+    count
   rescue Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge, Faraday::ConnectionFailed, ActiveRecord::LockWaitTimeout => error
     logger.info "[Elasticsearch] Error #{error.message} indexing DOIs created on #{options[:from_date]}."
 
@@ -560,6 +587,8 @@ def self.index_by_day(options={})
     end
 
     logger.info "[Elasticsearch] Indexed #{count} DOIs created on #{options[:from_date]}."
+
+    count
   end
 
   def self.index_by_ids(options={})
@@ -569,8 +598,9 @@ def self.index_by_ids(options={})
     # get every id between from_id and end_id
     (from_id..until_id).step(500).each do |id|
       DoiIndexByIdJob.perform_later(id: id)
-      puts "Queued indexing for DOIs with IDs starting with #{id}."
     end
+
+    (from_id..until_id).to_a.length
   end
 
   def self.index_by_id(options={})
@@ -602,6 +632,8 @@ def self.index_by_id(options={})
     elsif count > 0
       logger.info "[Elasticsearch] Indexed #{count} DOIs with IDs #{id} - #{(id + 499)}."
     end
+
+    count
   rescue Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge, Faraday::ConnectionFailed, ActiveRecord::LockWaitTimeout => error
     logger.info "[Elasticsearch] Error #{error.message} indexing DOIs with IDs #{id} - #{(id + 499)}."
 
@@ -613,6 +645,8 @@ def self.index_by_id(options={})
     end
 
     logger.info "[Elasticsearch] Indexed #{count} DOIs with IDs #{id} - #{(id + 499)}."
+
+    count
   end
 
   def uid
@@ -660,14 +694,13 @@ def client_id
 
   def client_id=(value)
     r = ::Client.where(symbol: value).first
-    #r = cached_client_response(value)
     fail ActiveRecord::RecordNotFound unless r.present?
 
     write_attribute(:datacentre, r.id)
   end
 
   def provider_id
-    provider.symbol.downcase
+    client.provider.symbol.downcase if client.present?
   end
 
   def prefix
@@ -736,7 +769,7 @@ def current_metadata
   end
 
   def metadata_version
-    fetch_cached_metadata_version
+    current_metadata ? current_metadata.metadata_version : 0
   end
 
   def current_media
@@ -849,6 +882,17 @@ def self.set_minted
 
   def self.transfer(options={})
     logger = Logger.new(STDOUT)
+
+    if options[:client_id].blank?
+      Logger.error "[Transfer] No client provided."
+      return nil
+    end
+
+    if options[:target_id].blank?
+      Logger.error "[Transfer] No target client provided."
+      return nil
+    end
+
     query = options[:query] || "*"
 
     response = Doi.query(query, client_id: options[:client_id], page: { size: 0, cursor: 1 })
@@ -869,6 +913,8 @@ def self.transfer(options={})
         end
       end
     end
+
+    response.results.total
   end
 
   # save to metadata table when xml has changed
diff --git a/app/models/provider_prefix.rb b/app/models/provider_prefix.rb
index 7d4629a69..5f9c641e0 100644
--- a/app/models/provider_prefix.rb
+++ b/app/models/provider_prefix.rb
@@ -34,7 +34,7 @@ def provider_id
 
   # workaround for non-standard database column names and association
   def provider_id=(value)
-    r = cached_provider_response(value)
+    r = Provider.where(symbol: value).first
     fail ActiveRecord::RecordNotFound unless r.present?
 
     self.allocator = r.id
diff --git a/app/models/user.rb b/app/models/user.rb
index cdfb5b789..22b95799d 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -62,13 +62,13 @@ def is_beta_tester?
   def provider
     return nil unless provider_id.present?
 
-    cached_provider_response(provider_id)
+    Provider.where(symbol: provider_id).first
   end
 
   def client
     return nil unless client_id.present?
 
-    cached_client_response(client_id)
+    ::Client.where(symbol: client_id).first
   end
 
   def self.reset(username)
diff --git a/lib/tasks/client.rake b/lib/tasks/client.rake
index 03033d5c5..bd714095c 100644
--- a/lib/tasks/client.rake
+++ b/lib/tasks/client.rake
@@ -161,12 +161,12 @@ namespace :client do
     end
 
     # update dois
-    Doi.transfer(from_date: "2011-01-01", client_id: client.id, target_id: target.id)
+    Doi.transfer(from_date: "2011-01-01", client_id: client.symbol, target_id: target.symbol)
 
     prefixes.each do |prefix|
-      provider_prefix = ProviderPrefix.create(provider: target.provider.symbol, prefix: prefix)
+      provider_prefix = ProviderPrefix.create(provider: target.provider, prefix: prefix)
       puts "Provider prefix for provider #{target.provider.symbol} and prefix #{prefix} created."
-      client_prefix = ClientPrefix.create(client: target.symbol, prefix: prefix, provider_prefix: provider_prefix.id)
+      client_prefix = ClientPrefix.create(client: target, prefix: prefix, provider_prefix: provider_prefix.id)
       puts "Client prefix for client #{target.symbol} and prefix #{prefix} created."
     end
   end
diff --git a/spec/concerns/facetable_spec.rb b/spec/concerns/facetable_spec.rb
index 7f3ce6d00..eabf0daea 100644
--- a/spec/concerns/facetable_spec.rb
+++ b/spec/concerns/facetable_spec.rb
@@ -1,45 +1,33 @@
 
-require 'rails_helper'
+# require 'rails_helper'
 
-describe 'Clients', type: :controller do
-  let(:provider) { create(:provider) }
-  let(:model) { ClientsController.new }
-  let!(:clients)  { create_list(:client, 5, provider: provider) }
-  let(:params)  { {year: 2008} }
-  let(:params2)  { {year: clients.first.created.year} }
+# describe 'Clients', type: :controller do
+#   let(:provider) { create(:provider) }
+#   let(:model) { ClientsController.new }
+#   let!(:clients)  { create_list(:client, 5, provider: provider) }
+#   let(:params)  { {year: 2008} }
+#   let(:params2)  { {year: clients.first.created.year} }
 
-  describe "facet by year activerecord" do
-    it "should return nothing" do
-      facet = model.client_year_facet params, Client
-      expect(facet.first[:count]).to eq(0)
-    end
-    it "should return all records" do
-      client = clients.first
-      facet = model.client_year_facet params2, Client
-      expect(facet.first[:count]).to eq(5)
-    end
-  end
+#   # describe "facet by year" do
+#   #   before do 
+#   #     Provider.create(provider)         
+#   #     clients.each { |item| Client.create(item) }
+#   #     sleep 2
+#   #   end
 
-  # describe "facet by year" do
-  #   before do 
-  #     Provider.create(provider)         
-  #     clients.each { |item| Client.create(item) }
-  #     sleep 2
-  #   end
+#   #   it "should return nothing" do
+#   #     puts Client.all
+#   #     facet = model.facet_by_year params, Client.all
+#   #     puts facet.class.name
+#   #     puts facet.inspect
+#   #     puts "chchc"
+#   #     expect(facet.first[:count]).to eq(0)
+#   #   end
 
-  #   it "should return nothing" do
-  #     puts Client.all
-  #     facet = model.facet_by_year params, Client.all
-  #     puts facet.class.name
-  #     puts facet.inspect
-  #     puts "chchc"
-  #     expect(facet.first[:count]).to eq(0)
-  #   end
-
-  #   it "should return all records" do
-  #     facet = model.facet_by_year params2, Client.all
-  #     puts facet
-  #     expect(facet.first[:count]).to eq(5)
-  #   end
-  # end
-end
+#   #   it "should return all records" do
+#   #     facet = model.facet_by_year params2, Client.all
+#   #     puts facet
+#   #     expect(facet.first[:count]).to eq(5)
+#   #   end
+#   # end
+# end
diff --git a/spec/concerns/indexable_spec.rb b/spec/concerns/indexable_spec.rb
index da0ec0a99..97a2ccb8d 100644
--- a/spec/concerns/indexable_spec.rb
+++ b/spec/concerns/indexable_spec.rb
@@ -19,98 +19,91 @@
   end
 end
 
-# describe "Indexable class methods", elasticsearch: true do
-#   context "client" do
-#     let!(:client) { create(:client) }
-
-#     it 'find_by_id' do
-#       sleep 1
-#       result = Client.find_by_id(client.symbol).results.first
-#       expect(result.symbol).to eq(client.symbol)
-#     end
-
-#     it 'find_by_ids' do
-#       sleep 1
-#       results = Client.find_by_ids(client.symbol).results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query' do
-#       sleep 1
-#       results = Client.query(client.name).results
-#       expect(results.total).to eq(1)
-#     end
-#   end
-
-#   context "provider" do
-#     let!(:provider) { create(:provider) }
-
-#     it 'find_by_id' do
-#       sleep 1
-#       result = Provider.find_by_id(provider.symbol).results.first
-#       expect(result.symbol).to eq(provider.symbol)
-#     end
-
-#     it 'find_by_ids' do
-#       sleep 1
-#       results = Provider.find_by_ids(provider.symbol).results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query' do
-#       sleep 1
-#       results = Provider.query(provider.name).results
-#       expect(results.total).to eq(1)
-#     end
-#   end
-
-#   context "doi" do
-#     let!(:doi) { create(:doi, title: "Soil investigations", publisher: "Pangaea", description: "this is a description") }
-
-#     it 'find_by_id' do
-#       sleep 1
-#       result = Doi.find_by_id(doi.doi).results.first
-#       expect(result.doi).to eq(doi.doi)
-#     end
-
-#     it 'query by doi' do
-#       sleep 1
-#       results = Doi.query(doi.doi).results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query by title' do
-#       sleep 1
-#       results = Doi.query("soil").results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query by publisher' do
-#       sleep 1
-#       results = Doi.query("pangaea").results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query by description' do
-#       sleep 1
-#       results = Doi.query("description").results
-#       expect(results.total).to eq(1)
-#     end
-
-#     it 'query by description not found' do
-#       sleep 1
-#       results = Doi.query("climate").results
-#       expect(results.total).to eq(0)
-#     end
-#   end
-
-  # context "prefix" do
-  #   let!(:prefix) { create(:prefix) }
-
-  #   it 'query' do
-  #     sleep 1
-  #     results = Prefix.query(prefix.prefix).results
-  #     expect(results.total).to eq(1)
-  #   end
-  # end
-# end
+describe "Indexable class methods", elasticsearch: true do
+  context "client" do
+    let!(:client) { create(:client) }
+
+    before do
+      Client.import
+      sleep 1
+    end
+
+    it 'find_by_id' do
+      result = Client.find_by_id(client.symbol).results.first
+      expect(result.symbol).to eq(client.symbol)
+    end
+
+    it 'find_by_ids' do
+      results = Client.find_by_ids(client.symbol).results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query' do
+      results = Client.query(client.name).results
+      expect(results.total).to eq(1)
+    end
+  end
+
+  context "provider" do
+    let!(:provider) { create(:provider) }
+
+    before do
+      Provider.import
+      sleep 1
+    end
+
+    it 'find_by_id' do
+      result = Provider.find_by_id(provider.symbol).results.first
+      expect(result.symbol).to eq(provider.symbol)
+    end
+
+    it 'find_by_ids' do
+      results = Provider.find_by_ids(provider.symbol).results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query' do
+      results = Provider.query(provider.name).results
+      expect(results.total).to eq(1)
+    end
+  end
+
+  context "doi" do
+    let!(:doi) { create(:doi, titles: { title: "Soil investigations" }, publisher: "Pangaea", descriptions: { description: "this is a description" }) }
+
+    before do
+      Doi.import
+      sleep 1
+    end
+
+    # it 'find_by_id' do
+    #   result = Doi.find_by_id(doi.doi).results.first
+    #   expect(result.doi).to eq(doi.doi)
+    # end
+
+    it 'query by doi' do
+      results = Doi.query(doi.doi).results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query by title' do
+      results = Doi.query("soil").results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query by publisher' do
+      results = Doi.query("pangaea").results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query by description' do
+      results = Doi.query("description").results
+      expect(results.total).to eq(1)
+    end
+
+    it 'query by description not found' do
+      results = Doi.query("climate").results
+      expect(results.total).to eq(0)
+    end
+  end
+end
diff --git a/spec/models/client_spec.rb b/spec/models/client_spec.rb
index 661d369ff..802fc5237 100644
--- a/spec/models/client_spec.rb
+++ b/spec/models/client_spec.rb
@@ -2,9 +2,7 @@
 
 describe Client, type: :model do
   let(:provider)  { create(:provider) }
-  let(:prefix)  { create(:prefix, prefix: "10.5072") }
   let(:client)  { create(:client, provider: provider) }
-  let(:target) { create(:client, provider: provider, symbol: provider.symbol + ".TARGET") }
 
   describe "Validations" do
     it { should validate_presence_of(:symbol) }
@@ -32,19 +30,4 @@
       expect(client.reload.symbol).to eq(client.symbol)
     end
   end
-
-  describe "doi transfer", elasticsearch: true do
-    let!(:dois) { create_list(:doi, 5, client: client) }
-
-    # it "transfer all DOIs" do
-    #   original_dois = Doi.where(client: client.symbol)
-    #   expect(Doi.where(datacentre: client.id).count).to eq(5)
-    #   expect(Doi.where(datacentre: target.id).count).to eq(0)
-    #   client.target_id = target.symbol
-    #   client.save
-    #   sleep 1
-    #   expect(Doi.where(datacentre: client.id).count).to eq(0)
-    #   expect(Doi.where(datacentre: target.id).count).to eq(5)
-    # end
-  end
 end
diff --git a/spec/models/doi_spec.rb b/spec/models/doi_spec.rb
index e991eca4c..9ee383efa 100644
--- a/spec/models/doi_spec.rb
+++ b/spec/models/doi_spec.rb
@@ -430,6 +430,94 @@
     end
   end
 
+  describe "import", elasticsearch: true do
+    let(:provider)  { create(:provider) }
+    let(:client)  { create(:client, provider: provider) }
+    let(:target) { create(:client, provider: provider, symbol: provider.symbol + ".TARGET", name: "Target Client") }
+    let!(:dois) { create_list(:doi, 3, client: client, creators: nil) }
+    let(:doi) { dois.first }
+
+    before do
+      Doi.import
+      sleep 1
+    end
+
+    it "import one" do
+      response = Doi.import_one(doi_id: doi.doi)
+      expect(response.schema_version).to eq("http://datacite.org/schema/kernel-4")
+    end
+
+    it "import all" do
+      response = Doi.import_all
+      expect(response).to eq(1)
+    end
+
+    it "import missing" do
+      response = Doi.import_missing
+      expect(response).to eq(1)
+    end
+
+    it "import by day" do
+      response = Doi.import_by_day(from_date: Date.today.strftime("%F"))
+      expect(response).to eq(3)
+    end
+
+    it "import by day missing" do
+      response = Doi.import_by_day_missing(from_date: Date.today.strftime("%F"))
+      expect(response).to eq(3)
+    end
+  end
+
+  describe "index", elasticsearch: true do
+    let(:provider)  { create(:provider) }
+    let(:client)  { create(:client, provider: provider) }
+    let(:target) { create(:client, provider: provider, symbol: provider.symbol + ".TARGET", name: "Target Client") }
+    let!(:dois) { create_list(:doi, 3, client: client, creators: nil) }
+    let(:doi) { dois.first }
+
+    before do
+      Doi.import
+      sleep 1
+    end
+
+    it "index" do
+      response = Doi.index
+      expect(response).to eq(1)
+    end
+
+    it "index by day" do
+      response = Doi.index_by_day(from_date: Date.today.strftime("%F"))
+      expect(response).to eq(3)
+    end
+
+    it "index by ids" do
+      response = Doi.index_by_ids
+      expect(response).to eq(500)
+    end
+
+    it "index by id" do
+      response = Doi.index_by_id(id: doi.id)
+      expect(response).to eq(3)
+    end
+  end
+
+  describe "transfer", elasticsearch: true do
+    let(:provider)  { create(:provider) }
+    let(:client)  { create(:client, provider: provider) }
+    let(:target) { create(:client, provider: provider, symbol: provider.symbol + ".TARGET", name: "Target Client") }
+    let!(:dois) { create_list(:doi, 3, client: client) }
+
+    before do
+      Doi.import
+      sleep 1
+    end
+
+    it "transfer all dois" do
+      response = Doi.transfer(client_id: client.symbol.downcase, target_id: target.symbol.downcase)
+      expect(response).to eq(3)
+    end
+  end
+
   describe "migrates landing page" do
     let(:provider)  { create(:provider, symbol: "ADMIN") }
     let(:client)  { create(:client, provider: provider) }
diff --git a/spec/requests/clients_spec.rb b/spec/requests/clients_spec.rb
index c2735bb5c..72ec2989f 100644
--- a/spec/requests/clients_spec.rb
+++ b/spec/requests/clients_spec.rb
@@ -1,7 +1,6 @@
 require 'rails_helper'
 
 describe 'Clients', type: :request, elasticsearch: true do
-  let!(:clients)  { create_list(:client, 10) }
   let(:ids) { clients.map { |c| c.uid }.join(",") }
   let(:bearer) { User.generate_token }
   let(:provider) { create(:provider, password_input: "12345") }
@@ -26,22 +25,24 @@
   let(:headers) { {'ACCEPT'=>'application/vnd.api+json', 'CONTENT_TYPE'=>'application/vnd.api+json', 'Authorization' => 'Bearer ' + bearer}}
   let(:query) { "jamon"}
 
-  # Test suite for GET /clients
-  # describe 'GET /clients', elasticsearch: true do
-  #   before do
-  #     sleep 1
-  #     get '/clients', headers: headers
-  #   end
+  describe 'GET /clients', elasticsearch: true do
+    let!(:clients)  { create_list(:client, 3) }
 
-  #   it 'returns clients' do
-  #     expect(json).not_to be_empty
-  #     expect(json['data'].size).to eq(11)
-  #   end
+    before do
+      Client.import
+      sleep 1
+      get '/clients', headers: headers
+    end
 
-  #   it 'returns status code 200' do
-  #     expect(response).to have_http_status(200)
-  #   end
-  # end
+    it 'returns clients' do
+      expect(json['data'].size).to eq(4)
+      expect(json.dig('meta', 'total')).to eq(4)
+    end
+
+    it 'returns status code 200' do
+      expect(response).to have_http_status(200)
+    end
+  end
 
   # # Test suite for GET /clients
   # describe 'GET /clients query' do
@@ -213,7 +214,6 @@
     before { delete "/clients/#{client.uid}", headers: headers }
 
     it 'returns status code 204' do
-      puts response.body
       expect(response).to have_http_status(204)
     end
 
@@ -230,19 +230,29 @@
     end
   end
 
-  # describe 'POST /clients/set-test-prefix' do
-  #   let(:prefix)  { create(:prefix, prefix: "10.5072") }
-  #   let!(:provider_psrefix)  { create(:provider_prefix, provider: provider, prefix: prefix) }
+  describe "doi transfer", elasticsearch: true do
+    let!(:dois) { create_list(:doi, 3, client: client) }
+    let(:target) { create(:client, provider: provider, symbol: provider.symbol + ".TARGET", name: "Target Client") }
+    let(:params) do
+      { "data" => { "type" => "clients",
+                    "attributes" => {
+                      "targetId" => target.symbol }} }
+    end
 
-  #   before { post '/clients/set-test-prefix', headers: headers }
+    before do
+      Doi.import
+      sleep 1
+      put "/clients/#{client.symbol}", params: params.to_json, headers: headers
+      sleep 1
+    end
 
-  #   it 'returns success' do
-  #     expect(json['message']).to eq("Test prefix added.")
-  #   end
+    it 'returns status code 200' do
+      expect(response).to have_http_status(200)
+    end
 
-  #   it 'returns status code 200' do
-  #     expect(response.body).to eq(2)
-  #     expect(response).to have_http_status(200)
-  #   end
-  # end
+    # it "transfered all DOIs" do
+    #   expect(Doi.query(nil, client_id: client.symbol.downcase).results.total).to eq(0)
+    #   expect(Doi.query(nil, client_id: target.symbol.downcase).results.total).to eq(3)
+    # end
+  end
 end
diff --git a/spec/requests/dois_spec.rb b/spec/requests/dois_spec.rb
index ad8ab115c..8f1f9a303 100644
--- a/spec/requests/dois_spec.rb
+++ b/spec/requests/dois_spec.rb
@@ -9,20 +9,24 @@
   let(:client) { create(:client, provider: provider, symbol: ENV['MDS_USERNAME'], password: ENV['MDS_PASSWORD']) }
   let!(:prefix) { create(:prefix, prefix: "10.14454") }
   let!(:client_prefix) { create(:client_prefix, client: client, prefix: prefix) }
-  let!(:dois) { create_list(:doi, 3, client: client) }
+  
   let(:doi) { create(:doi, client: client) }
   let(:bearer) { Client.generate_token(role_id: "client_admin", uid: client.symbol, provider_id: provider.symbol.downcase, client_id: client.symbol.downcase, password: client.password) }
   let(:headers) { { 'ACCEPT'=>'application/vnd.api+json', 'CONTENT_TYPE'=>'application/vnd.api+json', 'Authorization' => 'Bearer ' + bearer }}
 
   describe 'GET /dois', elasticsearch: true do
+    let!(:dois) { create_list(:doi, 3, client: client) }
+
     before do
+      Doi.import
       sleep 1
       get '/dois', headers: headers
     end
 
-    # it 'returns dois' do
-    #   expect(json['data'].size).to eq(0)
-    # end
+    it 'returns dois' do
+      expect(json['data'].size).to eq(3)
+      expect(json.dig('meta', 'total')).to eq(3)
+    end
 
     it 'returns status code 200' do
       expect(response).to have_http_status(200)
diff --git a/spec/requests/providers_spec.rb b/spec/requests/providers_spec.rb
index 4f8f8a343..b706ee5d6 100644
--- a/spec/requests/providers_spec.rb
+++ b/spec/requests/providers_spec.rb
@@ -1,9 +1,7 @@
 require 'rails_helper'
 
 describe "Providers", type: :request, elasticsearch: true  do
-  # initialize test data
-  let!(:providers)  { create_list(:provider, 10) }
-  let!(:provider) { providers.first }
+  let!(:provider) { create(:provider) }
   let(:token) { User.generate_token }
   let(:params) do
     { "data" => { "type" => "providers",
@@ -15,23 +13,28 @@
   end
   let(:headers) { {'ACCEPT'=>'application/vnd.api+json', 'CONTENT_TYPE'=>'application/vnd.api+json', 'Authorization' => 'Bearer ' + token } }
 
-  # # Test suite for GET /providers
-  # describe 'GET /providers' do
-  #   # make HTTP get request before each example
-  #   before { get '/providers', headers: headers }
+  describe 'GET /providers' do
+    let!(:providers)  { create_list(:provider, 3) }
 
-  #   it 'returns providers' do
-  #     expect(json['data'].size).to eq(10)
-  #   end
+    before do
+      Provider.import
+      sleep 1
+      get '/providers', headers: headers
+    end
 
-  #   it 'returns status code 200' do
-  #     expect(response).to have_http_status(200)
-  #   end
-  # end
+    it 'returns providers' do
+      expect(json['data'].size).to eq(4)
+      expect(json.dig('meta', 'total')).to eq(4)
+    end
+
+    it 'returns status code 200' do
+      expect(response).to have_http_status(200)
+    end
+  end
 
-  # Test suite for GET /providers/:id
   describe 'GET /providers/:id' do
     before { get "/providers/#{provider.symbol}" , headers: headers}
+    
     context 'when the record exists' do
       it 'returns the provider' do
         expect(json).not_to be_empty
diff --git a/spec/requests/works_spec.rb b/spec/requests/works_spec.rb
index 57c5dc04c..1af7fb301 100644
--- a/spec/requests/works_spec.rb
+++ b/spec/requests/works_spec.rb
@@ -16,13 +16,15 @@
 
   describe 'GET /works', elasticsearch: true do
     before do
+      Doi.import
       sleep 1
       get '/works', headers: headers
     end
 
-    # it 'returns dois' do
-    #   expect(json['data'].size).to eq(0)
-    # end
+    it 'returns dois' do
+      expect(json['data'].size).to eq(3)
+      expect(json.dig('meta', 'total')).to eq(3)
+    end
 
     it 'returns status code 200' do
       expect(response).to have_http_status(200)
diff --git a/spec/support/elasticsearch_helper.rb b/spec/support/elasticsearch_helper.rb
index 781720e5a..272f11844 100644
--- a/spec/support/elasticsearch_helper.rb
+++ b/spec/support/elasticsearch_helper.rb
@@ -2,7 +2,7 @@
 SEARCHABLE_MODELS = [Client, Provider, Doi]
 
 RSpec.configure do |config|
-  config.around :each, elasticsearch: true do |example|
+  config.around :all, elasticsearch: true do |example|
     SEARCHABLE_MODELS.each do |model|
       if Elasticsearch::Model.client.indices.exists? index: model.index_name
         model.__elasticsearch__.create_index! force: true