Merge pull request #512 from datacite/stats_portal_improvements

Trimmed down stats query

app/controllers/concerns/countable.rb

@@ -2,20 +2,20 @@ module Countable
   extend ActiveSupport::Concern
 
   included do
-    def doi_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: nil, state: nil)
+    def doi_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: nil)
       if client_id
-        response = Doi.query(nil, client_id: client_id, page: { number: 1, size: 0 })
+        response = Doi.stats_query(client_id: client_id)
       elsif provider_id
-        response = Doi.query(nil, provider_id: provider_id, page: { number: 1, size: 0 })
+        response = Doi.stats_query(provider_id: provider_id)
       elsif consortium_id
-        response = Doi.query(nil, consortium_id: consortium_id, page: { number: 1, size: 0 })
+        response = Doi.stats_query(consortium_id: consortium_id)
       elsif user_id
-        response = Doi.query(nil, user_id: user_id, state: state, page: { number: 1, size: 0 })
+        response = Doi.stats_query(user_id: user_id)
       else
-        response = Doi.query(nil, page: { number: 1, size: 0 })
+        response = Doi.stats_query
       end
 
-      response.results.total.positive? ? facet_by_year(response.response.aggregations.created.buckets) : []
+      response.results.total.positive? ? facet_by_year(response.aggregations.created.buckets) : []
     end
 
     def view_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: nil, state: nil)
@@ -31,7 +31,7 @@ def view_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: ni
         response = Doi.query(nil, page: { number: 1, size: 0 })
       end
 
-      response.results.total.positive? ? metric_facet_by_year(response.response.aggregations.views.buckets) : []
+      response.results.total.positive? ? metric_facet_by_year(response.aggregations.views.buckets) : []
     end
 
     def download_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: nil, state: nil)
@@ -47,7 +47,7 @@ def download_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id
         response = Doi.query(nil, page: { number: 1, size: 0 })
       end
 
-      response.results.total.positive? ? metric_facet_by_year(response.response.aggregations.downloads.buckets) : []
+      response.results.total.positive? ? metric_facet_by_year(response.aggregations.downloads.buckets) : []
     end
 
     def citation_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id: nil, state: nil)
@@ -63,7 +63,7 @@ def citation_count(client_id: nil, provider_id: nil, consortium_id: nil, user_id
         response = Doi.query(nil, page: { number: 1, size: 0 })
       end
 
-      response.results.total.positive? ? metric_facet_by_year(response.response.aggregations.citations.buckets) : []
+      response.results.total.positive? ? metric_facet_by_year(response.aggregations.citations.buckets) : []
     end
 
     # cumulative count clients by year
@@ -78,7 +78,7 @@ def client_count(provider_id: nil, consortium_id: nil)
         response = Client.query(nil, include_deleted: true, page: { number: 1, size: 0 })
       end 
 
-      response.results.total.positive? ? facet_by_cumulative_year(response.response.aggregations.cumulative_years.buckets) : []
+      response.results.total.positive? ? facet_by_cumulative_year(response.aggregations.cumulative_years.buckets) : []
     end
 
     # count active clients by provider. Provider can only be deleted when there are no active clients.
@@ -94,10 +94,10 @@ def active_client_count(provider_id: nil)
     def provider_count(consortium_id: nil)
       if consortium_id
         response = Provider.query(nil, consortium_id: consortium_id, include_deleted: true, page: { number: 1, size: 0 })
-        response.results.total.positive? ? facet_by_cumulative_year(response.response.aggregations.cumulative_years.buckets) : []
+        response.results.total.positive? ? facet_by_cumulative_year(response.aggregations.cumulative_years.buckets) : []
       else
         response = Provider.query(nil, include_deleted: true, page: { number: 1, size: 0 })
-        response.results.total.positive? ? facet_by_cumulative_year(response.response.aggregations.cumulative_years.buckets) : []
+        response.results.total.positive? ? facet_by_cumulative_year(response.aggregations.cumulative_years.buckets) : []
       end
     end
 
@@ -114,7 +114,7 @@ def resource_type_count(client_id: nil, provider_id: nil, consortium_id: nil, us
         response = Doi.query(nil, page: { number: 1, size: 0 })
       end
 
-      response.results.total.positive? ? facet_by_combined_key(response.response.aggregations.resource_types.buckets) : []
+      response.results.total.positive? ? facet_by_combined_key(response.aggregations.resource_types.buckets) : []
     end
   end
 end

app/controllers/providers_controller.rb

@@ -235,23 +235,23 @@ def stats
       providers = provider_count(consortium_id: nil)
       clients = client_count(provider_id: nil)
       dois = doi_count(provider_id: nil)
-      resource_types = resource_type_count(provider_id: nil)
+      # resource_types = resource_type_count(provider_id: nil)
       # citations = nil # citation_count(provider_id: nil)
       # views = nil # view_count(provider_id: nil)
       # downloads = nil # download_count(provider_id: nil)
     elsif @provider.member_type == "consortium"
       providers = provider_count(consortium_id: params[:id])
       clients = client_count(consortium_id: params[:id])
       dois = doi_count(consortium_id: params[:id])
-      resource_types = resource_type_count(consortium_id: params[:id])
+      # resource_types = resource_type_count(consortium_id: params[:id])
       # citations = citation_count(consortium_id: params[:id])
       # views = view_count(consortium_id: params[:id])
       # downloads = download_count(consortium_id: params[:id])
     else
       providers = nil
       clients = client_count(provider_id: params[:id])
       dois = doi_count(provider_id: params[:id])
-      resource_types = resource_type_count(provider_id: params[:id])
+      # resource_types = resource_type_count(provider_id: params[:id])
       # citations = citation_count(provider_id: params[:id])
       # views = view_count(provider_id: params[:id])
       # downloads = download_count(provider_id: params[:id])
@@ -261,7 +261,7 @@ def stats
       providers: providers,
       clients: clients,
       dois: dois,
-      "resourceTypes" => resource_types,
+      # "resourceTypes" => resource_types,
       # citations: citations,
       # views: views,
       # downloads: downloads,

app/models/doi.rb

@@ -675,6 +675,29 @@ def self.find_by_id(id)
     )
   end
 
+  def self.stats_query(options={})
+    filter = []
+    filter << { term: { provider_id: options[:provider_id] } } if options[:provider_id].present?
+    filter << { term: { client_id: options[:client_id] } } if options[:client_id].present?
+    filter << { term: { consortium_id: options[:consortium_id].upcase }} if options[:consortium_id].present?
+    filter << { term: { "creators.nameIdentifiers.nameIdentifier" => "https://orcid.org/#{orcid_from_url(options[:user_id])}" }} if options[:user_id].present?
+
+    aggregations = {
+      created: { date_histogram: { field: 'created', interval: 'year', format: 'year', order: { _key: "desc" }, min_doc_count: 1 },
+                 aggs: { bucket_truncate: { bucket_sort: { size: 12 } } } },
+    }
+
+    __elasticsearch__.search({
+      query: {
+        bool: {
+          must: [{ match_all: {} }],
+          filter: filter,
+        }
+      },
+      aggregations: aggregations,
+    })
+  end
+
   def self.query(query, options={})
     # support scroll api
     # map function is small performance hit

spec/concerns/countable_spec.rb

@@ -64,4 +64,65 @@
         {"count"=>2, "id"=>"2017", "title"=>"2017"}])
     end
   end
+
+  describe "doi_count" do
+    before do
+      allow(Time.zone).to receive(:now).and_return(Time.mktime(2015, 4, 8))
+    end
+
+    let(:consortium) { create(:provider, role_name: "ROLE_CONSORTIUM", symbol: "DC") }
+    let(:provider) { create(:provider, consortium: consortium, role_name: "ROLE_CONSORTIUM_ORGANIZATION", symbol: "DATACITE") }
+    let(:client) { create(:client, provider: provider, symbol: "DATACITE.TEST") }
+    let!(:dois) { create_list(:doi, 3, client: client, aasm_state: "findable") }
+    let!(:doi) { create(:doi) }
+
+    it "counts all dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count).to eq([{"count"=>4, "id"=>"2015", "title"=>"2015"}])
+    end
+
+    it "counts all consortium dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(consortium_id: "dc")).to eq([{"count"=>3, "id"=>"2015", "title"=>"2015"}])
+    end
+
+    it "counts all consortium dois no dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(consortium_id: "abc")).to eq([])
+    end
+
+    it "counts all provider dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(provider_id: "datacite")).to eq([{"count"=>3, "id"=>"2015", "title"=>"2015"}])
+    end
+
+    it "counts all provider dois no dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(provider_id: "abc")).to eq([])
+    end
+
+    it "counts all client dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(client_id: "datacite.test")).to eq([{"count"=>3, "id"=>"2015", "title"=>"2015"}])
+    end
+
+    it "counts all client dois no dois" do
+      Doi.import
+      sleep 2
+
+      expect(subject.doi_count(client_id: "abc")).to eq([])
+    end
+  end
 end

spec/models/doi_spec.rb

@@ -872,4 +872,81 @@
       expect(changed_doi.landing_page).to eq(landing_page)
     end
   end
+
+  describe "stats_query", elasticsearch: true do
+    subject { Doi }
+
+    before do
+      allow(Time.zone).to receive(:now).and_return(Time.mktime(2015, 4, 8))
+    end
+
+    let(:consortium) { create(:provider, role_name: "ROLE_CONSORTIUM", symbol: "DC") }
+    let(:provider) { create(:provider, consortium: consortium, role_name: "ROLE_CONSORTIUM_ORGANIZATION", symbol: "DATACITE") }
+    let(:client) { create(:client, provider: provider, symbol: "DATACITE.TEST") }
+    let!(:dois) { create_list(:doi, 3, client: client, aasm_state: "findable") }
+    let!(:doi) { create(:doi) }
+
+    it "counts all dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query
+      expect(response.results.total).to eq(4)
+      expect(response.aggregations.created.buckets).to eq([{"doc_count"=>4, "key"=>1420070400000, "key_as_string"=>"2015"}])
+    end
+
+    it "counts all consortia dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(consortium_id: "dc")
+      expect(response.results.total).to eq(3)
+      expect(response.aggregations.created.buckets).to eq([{"doc_count"=>3, "key"=>1420070400000, "key_as_string"=>"2015"}])
+    end
+
+    it "counts all consortia dois no dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(consortium_id: "abc")
+      expect(response.results.total).to eq(0)
+      expect(response.aggregations.created.buckets).to eq([])
+    end
+
+    it "counts all provider dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(provider_id: "datacite")
+      expect(response.results.total).to eq(3)
+      expect(response.aggregations.created.buckets).to eq([{"doc_count"=>3, "key"=>1420070400000, "key_as_string"=>"2015"}])
+    end
+
+    it "counts all provider dois no dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(provider_id: "abc")
+      expect(response.results.total).to eq(0)
+      expect(response.aggregations.created.buckets).to eq([])
+    end
+
+    it "counts all client dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(client_id: "datacite.test")
+      expect(response.results.total).to eq(3)
+      expect(response.aggregations.created.buckets).to eq([{"doc_count"=>3, "key"=>1420070400000, "key_as_string"=>"2015"}])
+    end
+
+    it "counts all client dois no dois" do
+      Doi.import
+      sleep 2
+
+      response = subject.stats_query(client_id: "datacite.abc")
+      expect(response.results.total).to eq(0)
+      expect(response.aggregations.created.buckets).to eq([])
+    end
+  end
 end

spec/requests/providers_spec.rb

@@ -145,7 +145,7 @@
 
       expect(last_response.status).to eq(200)
       expect(json["clients"]).to eq([{"count"=>1, "id"=>"2020", "title"=>"2020"}])
-      expect(json["resourceTypes"]).to eq([{"count"=>3, "id"=>"dataset", "title"=>"Dataset"}])
+      # expect(json["resourceTypes"]).to eq([{"count"=>3, "id"=>"dataset", "title"=>"Dataset"}])
       expect(json["dois"]).to eq([{"count"=>3, "id"=>"2020", "title"=>"2020"}])
     end
   end