From 5f7ea1b1df5dcf5056c238d5b87c8a4ae472195c Mon Sep 17 00:00:00 2001
From: Martin Fenner <martin.fenner@datacite.org>
Date: Thu, 26 Nov 2020 13:58:01 +0100
Subject: [PATCH] show missing dois in repository export. #676

---
 app/controllers/exports_controller.rb | 13 ++++++--
 app/models/client.rb                  | 47 ++++++++++++++-------------
 spec/requests/exports_spec.rb         |  8 +++--
 3 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/app/controllers/exports_controller.rb b/app/controllers/exports_controller.rb
index 7d42c2099..d5681859f 100644
--- a/app/controllers/exports_controller.rb
+++ b/app/controllers/exports_controller.rb
@@ -389,10 +389,15 @@ def repositories
       "doisCurrentYear",
       "doisPreviousYear",
       "doisTotal",
+      "doisDbTotal",
+      "doisMissing"
     ]
 
     csv = headers.to_csv
 
+    # get doi counts from database
+    dois_by_client = DataciteDoi.group(:datacentre).count
+
     clients.each do |client|
       # Limit for salesforce default of max 80 chars
       name =
@@ -400,6 +405,9 @@ def repositories
       # Clean the name to remove quotes, which can break csv parsers
       name.gsub!(/["']/, "")
 
+      db_total = dois_by_client[client.id.to_i].to_i
+      es_total = client_totals[client.uid] ? client_totals[client.uid]["count"] : 0
+
       row = {
         accountName: name,
         fabricaAccountId: client.symbol,
@@ -435,8 +443,9 @@ def repositories
           else
             0
           end,
-        doisCountTotal:
-          client_totals[client.uid] ? client_totals[client.uid]["count"] : 0,
+        doisCountTotal: es_total,
+        doisDbTotal: db_total,
+        doisMissing: db_total - es_total,
       }.values
 
       csv += CSV.generate_line row
diff --git a/app/models/client.rb b/app/models/client.rb
index 76892b4c8..fc5232a07 100644
--- a/app/models/client.rb
+++ b/app/models/client.rb
@@ -669,37 +669,38 @@ def self.export_doi_counts(query: nil)
     dois_by_client = DataciteDoi.group(:datacentre).count
     rows =
       clients.reduce([]) do |sum, client|
-        db_total = dois_by_client.dig(client.id).to_i
+        db_total = dois_by_client[client.id.to_i].to_i
         es_total =
           client_totals[client.uid] ? client_totals[client.uid]["count"] : 0
-        # if (db_total - es_total) > 0
-        # Limit for salesforce default of max 80 chars
-        name = +client.name.truncate(80)
-        # Clean the name to remove quotes, which can break csv parsers
-        name.gsub!(/["']/, "")
-
-        row = {
-          accountName: name,
-          fabricaAccountId: client.symbol,
-          parentFabricaAccountId:
-            client.provider.present? ? client.provider.symbol : nil,
-          doisCountTotal: db_total,
-          doisMissing: db_total - es_total,
-        }.values
-
-        puts CSV.generate_line(row)
-
-        sum << CSV.generate_line(row)
-        # end
+        if (db_total - es_total) > 0
+          # Limit for salesforce default of max 80 chars
+          name = +client.name.truncate(80)
+          # Clean the name to remove quotes, which can break csv parsers
+          name.gsub!(/["']/, "")
+
+          row = {
+            accountName: name,
+            fabricaAccountId: client.symbol,
+            parentFabricaAccountId:
+              client.provider.present? ? client.provider.symbol : nil,
+            doisCountTotal: es_total,
+            doisDbTotal: db_total,
+            doisMissing: db_total - es_total,
+          }.values
+
+          puts CSV.generate_line(row)
+
+          sum << CSV.generate_line(row)
+        end
 
         sum
       end
 
-    csv = CSV::Table.new(rows, headers: headers)
+    csv = [headers] + rows
 
-    logger.warn "Found #{csv.count} repositories with missing DOIs."
+    logger.warn "Found #{csv.size} repositories with missing DOIs."
 
-    csv.to_csv
+    csv.join(",")
   end
 
   protected
diff --git a/spec/requests/exports_spec.rb b/spec/requests/exports_spec.rb
index ba2b71675..f5fa908d9 100644
--- a/spec/requests/exports_spec.rb
+++ b/spec/requests/exports_spec.rb
@@ -92,8 +92,10 @@
       expect(csv[1]).to start_with(
         "University of Virginia Library,UVA.LIBRARY,UVA,true",
       )
-      dois_total = csv[1].strip.split(",").last.to_i
+      dois_total = csv[1].strip.split(",")[15].to_i
       expect(dois_total).to eq(3)
+      dois_missing = csv[1].strip.split(",")[17].to_i
+      expect(dois_missing).to eq(0)
     end
 
     it "returns repositories from date", vcr: false do
@@ -109,8 +111,10 @@
       expect(csv[1]).to start_with(
         "University of Virginia Library,UVA.LIBRARY,UVA,true",
       )
-      dois_total = csv[1].strip.split(",").last.to_i
+      dois_total = csv[1].strip.split(",")[15].to_i
       expect(dois_total).to eq(3)
+      dois_missing = csv[1].strip.split(",")[17].to_i
+      expect(dois_missing).to eq(0)
     end
   end