From 32402e494aaa39d5cc6488b227bd5b28edc3587f Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 31 Dec 2018 12:00:04 -0600 Subject: [PATCH 1/7] methods for stats --- app/controllers/clients_controller.rb | 16 ++++++++++++++++ app/controllers/providers_controller.rb | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index 3fccb4bb3..b18d06521 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -127,6 +127,22 @@ def destroy end end + def totals + page ={size: 25, number: 1} + ttl = Client.all.map do |client| + response = Doi.query("", client_id: client.symbol.downcase, page: page) + total = response.results.total + states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil + temporal ={} + temporal[:last_thirty] = total > 0 ? facet_by_key(response.response.aggregations.last_thirty_days.buckets) : nil + temporal[:last_year] = total > 0 ? facet_by_key(response.response.aggregations.last_year.buckets) : nil + temporal[:last_two_year] = total > 0 ? facet_by_key(response.response.aggregations.last_two_year.buckets) : nil + id = client.symbol + {id: id, title: id, count: total, states: states, temporal: temporal} + end + render json: ttl, status: :ok + end + protected def set_include diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index 6d5cc1eeb..6965d412e 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -120,6 +120,26 @@ def update end end + def totals + page ={size: 25, number: 1} + logger = Logger.new(STDOUT) + + ttl = Provider.all.map do |provider| + response = Doi.query("", provider_id: provider.symbol.downcase, page: page) + logger.info response.results.total + total = response.results.total + states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil + temporal ={} + temporal[:last_thirty] = total > 0 ? facet_by_key(response.response.aggregations.last_thirty_days.buckets) : nil + temporal[:last_year] = total > 0 ? facet_by_key(response.response.aggregations.last_year.buckets) : nil + temporal[:last_two_year] = total > 0 ? facet_by_key(response.response.aggregations.last_two_year.buckets) : nil + id = provider.symbol + {id: id, title: id, count: total, states: states, temporal: temporal} + end + render json: ttl, status: :ok + end + + # don't delete, but set deleted_at timestamp # a provider with clients or prefixes can't be deleted def destroy From f0ae73758b382f43b06a6d27d8ea619be06fe1ab Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 31 Dec 2018 12:00:14 -0600 Subject: [PATCH 2/7] temporal aggreagtions --- app/models/doi.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/doi.rb b/app/models/doi.rb index 9230b74c6..e314a4034 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -316,6 +316,9 @@ def self.query_aggregations link_checks_citation_doi: { value_count: { field: "landing_page.citationDoi" } }, links_checked: { value_count: { field: "landing_page.checked" } }, sources: { terms: { field: 'source', size: 15, min_doc_count: 1 } }, + last_thirty_days: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-30d/d", to: "now/d"} } }, + last_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-1y/y", to: "now/d"} } }, + last_two_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-2y/y", to: "now/d"} } }, } end From 29a6aeaa0bd455b6e709c8e4829e76db386384c3 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Mon, 31 Dec 2018 12:00:22 -0600 Subject: [PATCH 3/7] new routes --- config/routes.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/routes.rb b/config/routes.rb index 063f71ce3..2bc26fb08 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -39,6 +39,8 @@ get 'dois/random', :to => 'dois#random' get 'dois/:id/get-url', :to => 'dois#get_url', constraints: { :id => /.+/ } get 'dois/get-dois', :to => 'dois#get_dois' + get 'providers/totals', :to => 'providers#totals' + get 'clients/totals', :to => 'clients#totals' # manage prefixes, keep database in sync for changes via MDS post 'client-prefixes/set-created', :to => 'client_prefixes#set_created' From 76f847c35c7209ee0897ef126d7dc00e0164c2c0 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 8 Jan 2019 13:38:15 +0100 Subject: [PATCH 4/7] formatting facets --- app/controllers/concerns/facetable.rb | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/controllers/concerns/facetable.rb b/app/controllers/concerns/facetable.rb index 623febd63..85eddd55d 100644 --- a/app/controllers/concerns/facetable.rb +++ b/app/controllers/concerns/facetable.rb @@ -22,6 +22,22 @@ def facet_by_year(arr) end end + def facet_anual(arr) + arr.map do |hsh| + { "id" => hsh["key"][0..3], + "title" => hsh["key"][0..3], + "count" => hsh["doc_count"] } + end + end + + def facet_by_date(arr) + arr.map do |hsh| + { "id" => hsh["key"][0..9], + "title" => hsh["key"][0..9], + "count" => hsh["doc_count"] } + end + end + def facet_by_cumulative_year(arr) arr.map do |hsh| { "id" => hsh["key"].to_s, @@ -118,6 +134,20 @@ def facet_by_client(arr) "count" => hsh["doc_count"] } end end + + # def get_all_providers_aggs + # page = { size: 25, number: 1} + # response = Doi.query("", page: page) + # after = response.response.aggregations.providers_x.after_key.doi ||="" + # aggs = response.response.aggregations.providers_x.buckets + # loop do + # resp = Doi.query("", {after_key: after }) + # aggs = aggs.concat resp.response.aggregations.providers_x.buckets + # after = response.response.aggregations.providers_x.after_key.doi + # break if after.nil? + # end + # aggs + # end end end From 7bed7866670b5c89b238eeb452d157324f70f72b Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 8 Jan 2019 13:38:28 +0100 Subject: [PATCH 5/7] change aggregations --- app/models/doi.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/doi.rb b/app/models/doi.rb index e314a4034..6cee7b878 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -299,6 +299,10 @@ def as_indexed_json(options={}) end def self.query_aggregations + beginning_of_year = DateTime.current.beginning_of_year.strftime('%Y-%m-%d') + beginning_of_last_year = DateTime.current.beginning_of_year.last_year.strftime('%Y-%m-%d') + beginning_of_month = DateTime.current.beginning_of_month.strftime('%Y-%m-%d') + { resource_types: { terms: { field: 'types.resourceTypeGeneral', size: 15, min_doc_count: 1 } }, states: { terms: { field: 'aasm_state', size: 15, min_doc_count: 1 } }, @@ -316,9 +320,9 @@ def self.query_aggregations link_checks_citation_doi: { value_count: { field: "landing_page.citationDoi" } }, links_checked: { value_count: { field: "landing_page.checked" } }, sources: { terms: { field: 'source', size: 15, min_doc_count: 1 } }, - last_thirty_days: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-30d/d", to: "now/d"} } }, - last_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-1y/y", to: "now/d"} } }, - last_two_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: "now-2y/y", to: "now/d"} } }, + this_month: { date_range: { field: 'created', time_zone: "CET", ranges: {from: beginning_of_month, to: "now/d"} } }, + this_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: beginning_of_year, to: "now/d"} } }, + last_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: beginning_of_last_year, to: beginning_of_year} } } } end From c5fdb2bed524976d9f38c0987de1f070c82e0e81 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 8 Jan 2019 13:38:51 +0100 Subject: [PATCH 6/7] new facets --- app/controllers/clients_controller.rb | 10 ++++++---- app/controllers/providers_controller.rb | 14 +++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index b18d06521..e32f366fa 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -129,14 +129,16 @@ def destroy def totals page ={size: 25, number: 1} - ttl = Client.all.map do |client| + page_prov = { size: 2000, number: 1} + + ttl = Client.query("", page: page_prov).map do |client| response = Doi.query("", client_id: client.symbol.downcase, page: page) total = response.results.total states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil temporal ={} - temporal[:last_thirty] = total > 0 ? facet_by_key(response.response.aggregations.last_thirty_days.buckets) : nil - temporal[:last_year] = total > 0 ? facet_by_key(response.response.aggregations.last_year.buckets) : nil - temporal[:last_two_year] = total > 0 ? facet_by_key(response.response.aggregations.last_two_year.buckets) : nil + temporal[:this_month] = total > 0 ? facet_by_date(response.response.aggregations.this_month.buckets) : nil + temporal[:this_year] = total > 0 ? facet_anual(response.response.aggregations.this_year.buckets) : nil + temporal[:last_year] = total > 0 ? facet_anual(response.response.aggregations.last_year.buckets) : nil id = client.symbol {id: id, title: id, count: total, states: states, temporal: temporal} end diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index 6965d412e..85b5bd950 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -121,18 +121,18 @@ def update end def totals - page ={size: 25, number: 1} - logger = Logger.new(STDOUT) + page = { size: 25, number: 1} + page_prov = { size: 500, number: 1} - ttl = Provider.all.map do |provider| + ttl = Provider.query("", page: page_prov).map do |provider| response = Doi.query("", provider_id: provider.symbol.downcase, page: page) - logger.info response.results.total total = response.results.total states = total > 0 ? facet_by_key(response.response.aggregations.states.buckets) : nil temporal ={} - temporal[:last_thirty] = total > 0 ? facet_by_key(response.response.aggregations.last_thirty_days.buckets) : nil - temporal[:last_year] = total > 0 ? facet_by_key(response.response.aggregations.last_year.buckets) : nil - temporal[:last_two_year] = total > 0 ? facet_by_key(response.response.aggregations.last_two_year.buckets) : nil + temporal[:this_month] = total > 0 ? facet_by_date(response.response.aggregations.this_month.buckets) : nil + temporal[:this_year] = total > 0 ? facet_anual(response.response.aggregations.this_year.buckets) : nil + temporal[:last_year] = total > 0 ? facet_anual(response.response.aggregations.last_year.buckets) : nil + # temporal[:providers] = total > 0 ? facet_by_key(response.response.aggregations.provider_x.buckets) : nil id = provider.symbol {id: id, title: id, count: total, states: states, temporal: temporal} end From 75415b83bcb758a6882480e41abba3cfd80208b4 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 8 Jan 2019 13:40:07 +0100 Subject: [PATCH 7/7] spaces --- app/controllers/clients_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index e32f366fa..c1f8373cb 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -128,8 +128,8 @@ def destroy end def totals - page ={size: 25, number: 1} - page_prov = { size: 2000, number: 1} + page = { size: 25, number: 1 } + page_prov = { size: 2000, number: 1 } ttl = Client.query("", page: page_prov).map do |client| response = Doi.query("", client_id: client.symbol.downcase, page: page)