support pagination in handle server get requests. #194

app/models/concerns/helpable.rb

@@ -109,19 +109,32 @@ module ClassMethods
     def get_dois(options={})
       return OpenStruct.new(body: { "errors" => [{ "title" => "Prefix missing" }] }) unless options[:prefix].present?
 
-      url = "#{ENV['HANDLE_URL']}/api/handles?prefix=#{options[:prefix]}"
-      response = Maremma.get(url, username: "300%3A#{ENV['HANDLE_USERNAME']}", password: ENV['HANDLE_PASSWORD'], ssl_self_signed: true, timeout: 10)
-
-      if response.status == 200
-        response
-      else
-        text = "Error " + response.body["errors"].inspect
-
-        logger = Logger.new(STDOUT)
-        logger.error "[Handle] " + text
-        User.send_notification_to_slack(text, title: "Error #{response.status.to_s}", level: "danger") unless Rails.env.test?
-        response
+      count_url = "#{ENV['HANDLE_URL']}/api/handles?prefix=#{options[:prefix]}&pageSize=0"
+      response = Maremma.get(count_url, username: "300%3A#{ENV['HANDLE_USERNAME']}", password: ENV['HANDLE_PASSWORD'], ssl_self_signed: true, timeout: 10)
+
+      total = response.body.dig("data", "totalCount").to_i
+
+      if total > 0
+        # walk through paginated results
+        total_pages = (total.to_f / 1000).ceil
+
+        (0...total_pages).each do |page|
+          url = "#{ENV['HANDLE_URL']}/api/handles?prefix=#{options[:prefix]}&page=#{page}&pageSize=1000"
+          response = Maremma.get(url, username: "300%3A#{ENV['HANDLE_USERNAME']}", password: ENV['HANDLE_PASSWORD'], ssl_self_signed: true, timeout: 10)
+
+          if response.status == 200
+            puts (response.body.dig("data", "handles") || []).join("\n")
+          else
+            text = "Error " + response.body["errors"].inspect
+
+            logger = Logger.new(STDOUT)
+            logger.error "[Handle] " + text
+            User.send_notification_to_slack(text, title: "Error #{response.status.to_s}", level: "danger") unless Rails.env.test?
+          end
+        end
       end
+
+      puts "#{total} DOIs found."
     end
 
     def get_doi(options={})

config/initializers/_version.rb

@@ -1,5 +1,5 @@
 module Lupo
   class Application
-    VERSION = "2.3.13"
+    VERSION = "2.3.14"
   end
 end

lib/tasks/handle.rake

@@ -6,9 +6,7 @@ namespace :handle do
       exit
     end
 
-    response = Doi.get_dois(prefix: ENV['PREFIX'])
-    puts (response.body.dig("data", "handles") || []).join("\n")
-    puts "Found " + (response.body.dig("data", "totalCount") || "0") + " DOIs with prefix #{ENV['PREFIX']}."
+    Doi.get_dois(prefix: ENV['PREFIX'])
   end
 
   desc 'Get DOI'