support graphql batch requests. #274

datacite · May 16, 2019 · 7c2c733 · 7c2c733
1 parent 6819066
commit 7c2c733
Show file tree

Hide file tree

Showing 39 changed files with 134 additions and 101 deletions.
diff --git a/Gemfile b/Gemfile
@@ -58,8 +58,8 @@ gem 'audited', '~> 4.8'
 gem 'git', '~> 1.5'
 gem 'graphql', '~> 1.9', '>= 1.9.4'
 gem 'graphql-errors', '~> 0.3.0'
-# gem 'graphql-batch', '~> 0.4.0'
-# gem 'graphql-cache', '~> 0.6.0'
+gem 'graphql-batch', '~> 0.4.0'
+gem 'graphql-cache', '~> 0.6.0'
 
 group :development, :test do
   gem 'rspec-rails', '~> 3.8', '>= 3.8.2'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    aasm (5.0.3)
+    aasm (5.0.4)
       concurrent-ruby (~> 1.0)
     actioncable (5.2.3)
       actionpack (= 5.2.3)
@@ -58,21 +58,21 @@ GEM
     audited (4.8.0)
       activerecord (>= 4.0, < 5.3)
     aws-eventstream (1.0.3)
-    aws-partitions (1.160.0)
-    aws-sdk-core (3.50.0)
+    aws-partitions (1.162.0)
+    aws-sdk-core (3.52.1)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.18.0)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-sdk-kms (1.20.0)
+      aws-sdk-core (~> 3, >= 3.52.1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.36.1)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-sdk-s3 (1.38.0)
+      aws-sdk-core (~> 3, >= 3.52.1)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.0)
-    aws-sdk-sqs (1.13.0)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-sdk-sqs (1.15.0)
+      aws-sdk-core (~> 3, >= 3.52.1)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
@@ -127,13 +127,14 @@ GEM
     builder (3.2.3)
     byebug (11.0.1)
     cancancan (2.3.0)
-    capybara (3.19.0)
+    capybara (3.20.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (~> 1.2)
+      uglifier
       xpath (~> 3.2)
     case_transform (0.2)
       activesupport
@@ -201,6 +202,7 @@ GEM
       nokogiri (>= 1.4.3)
     erubi (1.8.0)
     excon (0.64.0)
+    execjs (2.7.0)
     facets (3.1.0)
     factory_bot (4.11.1)
       activesupport (>= 3.0.0)
@@ -231,6 +233,11 @@ GEM
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     graphql (1.9.4)
+    graphql-batch (0.4.0)
+      graphql (>= 1.3, < 2)
+      promise.rb (~> 0.7.2)
+    graphql-cache (0.6.0)
+      graphql (~> 1, > 1.8)
     graphql-errors (0.3.0)
       graphql (>= 1.6.0, < 2)
     haml (5.0.4)
@@ -321,7 +328,7 @@ GEM
       i18n (>= 0.6.4, <= 2)
     msgpack (1.2.10)
     multi_json (1.13.1)
-    multipart-post (2.1.0)
+    multipart-post (2.1.1)
     mysql2 (0.4.10)
     namae (1.0.1)
     netrc (0.11.0)
@@ -342,6 +349,7 @@ GEM
       addressable
       css_parser (>= 1.6.0)
       htmlentities (>= 4.0.0)
+    promise.rb (0.7.4)
     public_suffix (2.0.5)
     pwqgen.rb (0.1.0)
       docopt (~> 0.5)
@@ -404,7 +412,7 @@ GEM
       rdf (>= 2.2, < 4.0)
     rdf-xsd (3.0.1)
       rdf (~> 3.0)
-    regexp_parser (1.4.0)
+    regexp_parser (1.5.0)
     request_store (1.4.1)
       rack (>= 1.4)
     rest-client (2.0.2)
@@ -443,7 +451,7 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.6)
-    rubocop-performance (1.2.0)
+    rubocop-performance (1.3.0)
       rubocop (>= 0.68.0)
     ruby-enum (0.7.2)
       i18n
@@ -499,6 +507,8 @@ GEM
       tilt (>= 1.4, < 3)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    uglifier (4.1.20)
+      execjs (>= 0.3.0, < 3)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.6)
@@ -557,6 +567,8 @@ DEPENDENCIES
   gender_detector (~> 0.1.2)
   git (~> 1.5)
   graphql (~> 1.9, >= 1.9.4)
+  graphql-batch (~> 0.4.0)
+  graphql-cache (~> 0.6.0)
   graphql-errors (~> 0.3.0)
   iso8601 (~> 0.9.0)
   json (~> 1.8, >= 1.8.5)

diff --git a/app/controllers/activities_controller.rb b/app/controllers/activities_controller.rb
@@ -17,7 +17,7 @@ def index
     if params[:id].present?
       response = Activity.find_by_id(params[:id]) 
     elsif params[:ids].present?
-      response = Activity.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Activity.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Activity.query(params[:query], uid: params[:doi_id], page: page, sort: sort)
     end

diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb
@@ -23,7 +23,7 @@ def index
     if params[:id].present?
       response = Client.find_by_id(params[:id]) 
     elsif params[:ids].present?
-      response = Client.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Client.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Client.query(params[:query], year: params[:year], provider_id: params[:provider_id], software: params[:software], page: page, sort: sort)
     end

diff --git a/app/controllers/concerns/facetable.rb b/app/controllers/concerns/facetable.rb
@@ -88,7 +88,7 @@ def facet_by_provider(arr)
       # generate hash with id and name for each provider in facet
 
       ids = arr.map { |hsh| hsh["key"] }.join(",")
-      providers = Provider.find_by_ids(ids, size: 1000).records.pluck(:symbol, :name).to_h
+      providers = Provider.find_by_id(ids, size: 1000).records.pluck(:symbol, :name).to_h
 
       arr.map do |hsh|
         { "id" => hsh["key"],
@@ -163,7 +163,7 @@ def facet_by_provider_ids(arr)
     def facet_by_client(arr)
       # generate hash with id and name for each client in facet
       ids = arr.map { |hsh| hsh["key"] }.join(",")
-      clients = Client.find_by_ids(ids).records.pluck(:symbol, :name).to_h
+      clients = Client.find_by_id(ids).records.pluck(:symbol, :name).to_h
 
       arr.map do |hsh|
         { "id" => hsh["key"],

diff --git a/app/controllers/data_centers_controller.rb b/app/controllers/data_centers_controller.rb
@@ -17,7 +17,7 @@ def index
     if params[:id].present?
       response = Client.find_by_id(params[:id]) 
     elsif params[:ids].present?
-      response = Client.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Client.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Client.query(params[:query], year: params[:year], provider_id: params[:member_id], fields: params[:fields], page: page, sort: sort)
     end

diff --git a/app/controllers/dois_controller.rb b/app/controllers/dois_controller.rb
@@ -46,7 +46,7 @@ def index
         response = Doi.find_by_id(params[:id])
       }.to_s + " ms"
     elsif params[:ids].present?
-      response = Doi.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Doi.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Doi.query(params[:query],
                           state: params[:state],

diff --git a/app/controllers/members_controller.rb b/app/controllers/members_controller.rb
@@ -16,7 +16,7 @@ def index
     if params[:id].present?
       response = Provider.find_by_id(params[:id])
     elsif params[:ids].present?
-      response = Provider.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Provider.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Provider.query(params[:query], all_members: true, year: params[:year], region: params[:region], organization_type: params[:organization_type], focus_area: params[:focus_area], fields: params[:fields], page: page, sort: sort)
     end

diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb
@@ -23,7 +23,7 @@ def index
     if params[:id].present?
       response = Provider.find_by_id(params[:id])
     elsif params[:ids].present?
-      response = Provider.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Provider.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Provider.query(params[:query], year: params[:year], region: params[:region], organization_type: params[:organization_type], focus_area: params[:focus_area], page: page, sort: sort)
     end

diff --git a/app/controllers/works_controller.rb b/app/controllers/works_controller.rb
@@ -30,7 +30,7 @@ def index
     if params[:id].present?
       response = Doi.find_by_id(params[:id])
     elsif params[:ids].present?
-      response = Doi.find_by_ids(params[:ids], page: page, sort: sort)
+      response = Doi.find_by_id(params[:ids], page: page, sort: sort)
     else
       response = Doi.query(params[:query],
                           state: "findable",

diff --git a/app/graphql/active_record_loader.rb b/app/graphql/active_record_loader.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class ActiveRecordLoader < GraphQL::Batch::Loader
+  def initialize(model)
+    @model = model
+  end
+
+  def perform(ids)
+    @model.where(id: ids).each { |record| fulfill(record.id, record) }
+    ids.each { |id| fulfill(id, nil) unless fulfilled?(id) }
+  end
+end
diff --git a/app/graphql/elasticsearch_loader.rb b/app/graphql/elasticsearch_loader.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class ElasticsearchLoader < GraphQL::Batch::Loader
+  def initialize(model)
+    @model = model
+  end
+
+  def perform(ids)
+    @model.find_by_id(ids).results.each { |record| fulfill(record.uid, record) }
+    ids.each { |id| fulfill(id, nil) unless fulfilled?(id) }
+  end
+end
diff --git a/app/graphql/lupo_schema.rb b/app/graphql/lupo_schema.rb
@@ -6,6 +6,9 @@ class LupoSchema < GraphQL::Schema
 
   # mutation(Types::MutationType)
   query(QueryType)
+
+  use GraphQL::Batch
+  use GraphQL::Cache
 end
 
 GraphQL::Errors.configure(LupoSchema) do

diff --git a/app/graphql/types/base_connection.rb b/app/graphql/types/base_connection.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class BaseConnection < GraphQL::Types::Relay::BaseConnection
+end
diff --git a/app/graphql/types/base_interface.rb b/app/graphql/types/base_interface.rb
@@ -2,4 +2,6 @@
 
 module BaseInterface
   include GraphQL::Schema::Interface
+
+  field_class GraphQL::Cache::Field
 end
diff --git a/app/graphql/types/base_object.rb b/app/graphql/types/base_object.rb
@@ -1,4 +1,5 @@
 # frozen_string_literal: true
 
 class BaseObject < GraphQL::Schema::Object
+  field_class GraphQL::Cache::Field
 end
diff --git a/app/graphql/types/client_connection_with_meta_type.rb b/app/graphql/types/client_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ClientConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class ClientConnectionWithMetaType < BaseConnection
   edge_type(ClientEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/client_dataset_connection_with_meta_type.rb b/app/graphql/types/client_dataset_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ClientDatasetConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class ClientDatasetConnectionWithMetaType < BaseConnection
   edge_type(DatasetEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/client_publication_connection_with_meta_type.rb b/app/graphql/types/client_publication_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ClientPublicationConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class ClientPublicationConnectionWithMetaType < BaseConnection
   edge_type(DatasetEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/client_software_connection_with_meta_type.rb b/app/graphql/types/client_software_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ClientSoftwareConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class ClientSoftwareConnectionWithMetaType < BaseConnection
   edge_type(DatasetEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/dataset_connection_with_meta_type.rb b/app/graphql/types/dataset_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class DatasetConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class DatasetConnectionWithMetaType < BaseConnection
   edge_type(DatasetEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/funder_connection_with_meta_type.rb b/app/graphql/types/funder_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class FunderConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class FunderConnectionWithMetaType < BaseConnection
   edge_type(FunderEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/funder_dataset_connection_with_meta_type.rb b/app/graphql/types/funder_dataset_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class FunderDatasetConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class FunderDatasetConnectionWithMetaType < BaseConnection
   edge_type(EventDataEdgeType, edge_class: EventDataEdge)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/funder_publication_connection_with_meta_type.rb b/app/graphql/types/funder_publication_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class FunderPublicationConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class FunderPublicationConnectionWithMetaType < BaseConnection
   edge_type(EventDataEdgeType, edge_class: EventDataEdge)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/funder_software_connection_with_meta_type.rb b/app/graphql/types/funder_software_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class FunderSoftwareConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class FunderSoftwareConnectionWithMetaType < BaseConnection
   edge_type(EventDataEdgeType, edge_class: EventDataEdge)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/funder_type.rb b/app/graphql/types/funder_type.rb
@@ -24,22 +24,22 @@ class FunderType < BaseObject
   def datasets(**args)
     ids = Event.query(nil, obj_id: object[:id], citation_type: "Dataset-Funder").fetch(:data, []).map do |e|
       doi_from_url(e[:subj_id])
-    end.join(",")
-    Doi.find_by_ids(ids, page: { number: 1, size: args[:first] }).to_a
+    end
+    ElasticsearchLoader.for(Doi).load_many(ids)
   end
 
   def publications(**args)
     ids = Event.query(nil, obj_id: object[:id], citation_type: "Funder-ScholarlyArticle").fetch(:data, []).map do |e|
       doi_from_url(e[:subj_id])
-    end.join(",")
-    Doi.find_by_ids(ids, page: { number: 1, size: args[:first] }).to_a
+    end
+    ElasticsearchLoader.for(Doi).load_many(ids)
   end
 
   def softwares(**args)
     ids = Event.query(nil, obj_id: object[:id], citation_type: "Funder-SoftwareSourceCode").fetch(:data, []).map do |e|
       doi_from_url(e[:subj_id])
-    end.join(",")
-    Doi.find_by_ids(ids, page: { number: 1, size: args[:first] }).to_a
+    end
+    ElasticsearchLoader.for(Doi).load_many(ids)
   end
 
   def doi_from_url(url)

diff --git a/app/graphql/types/organization_connection_with_meta_type.rb b/app/graphql/types/organization_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class OrganizationConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class OrganizationConnectionWithMetaType < BaseConnection
   edge_type(OrganizationEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/prefix_connection_with_meta_type.rb b/app/graphql/types/prefix_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class PrefixConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class PrefixConnectionWithMetaType < BaseConnection
   edge_type(PrefixEdgeType)
 
   field :total_count, Integer, null: false

diff --git a/app/graphql/types/provider_connection_with_meta_type.rb b/app/graphql/types/provider_connection_with_meta_type.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ProviderConnectionWithMetaType < GraphQL::Types::Relay::BaseConnection
+class ProviderConnectionWithMetaType < BaseConnection
   edge_type(ProviderEdgeType)
 
   field :total_count, Integer, null: false