From 910d8068108c516a66ec51cdff6e8d119316f47f Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Mon, 30 Sep 2019 15:59:37 +0200 Subject: [PATCH 1/8] Add new reports endpoint --- app/controllers/reports_controller.rb | 279 ++++++++++++++++++++++++++ config/routes.rb | 5 + 2 files changed, 284 insertions(+) create mode 100644 app/controllers/reports_controller.rb diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb new file mode 100644 index 000000000..5b968999f --- /dev/null +++ b/app/controllers/reports_controller.rb @@ -0,0 +1,279 @@ +class ReportsController < ApplicationController + include ActionController::MimeResponds + + + def contacts + begin + + # Loop through all providers + providers = [] + + page = { size: 1000, number: 1} + response = Provider.query(nil, page: page, include_deleted: true) + providers = providers + response.records.to_a + + total = response.results.total + total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 + + # keep going for all pages + page_num = 2 + while page_num <= total_pages + page = { size: 1000, number: page_num } + response = Provider.query(nil, page: page) + providers = providers + response.records.to_a + page_num += 1 + end + + respond_to do |format| + format.csv do + headers = %W( + fabricaAccountId + email, + firstName, + lastName, + type + ) + + csv = headers.to_csv + + providers.each do |provider| + + csv += CSV.generate_line [ + provider.symbol, + provider.technical_contact_email, + provider.technical_contact_given_name, + provider.technical_contact_family_name, + 'technical' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.secondary_technical_contact_email, + provider.secondary_technical_contact_given_name, + provider.secondary_technical_contact_family_name, + 'secondaryTechnicalContact' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.service_contact_email, + provider.service_contact_given_name, + provider.service_contact_family_name, + 'service' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.secondary_service_contact_email, + provider.secondary_service_contact_given_name, + provider.secondary_service_contact_family_name, + 'secondaryService' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.voting_contact_email, + provider.voting_contact_given_name, + provider.voting_contact_family_name, + 'voting' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.billing_contact_email, + provider.billing_contact_given_name, + provider.billing_contact_family_name, + 'billing' + ] + csv += CSV.generate_line [ + provider.symbol, + provider.secondary_billing_contact_email, + provider.secondary_billing_contact_given_name, + provider.secondary_billing_contact_family_name, + 'secondaryBilling' + ] + + end + + send_data csv, filename: "contacts-#{Date.today}.csv" + end + end + + rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception + Raven.capture_exception(exception) + + message = JSON.parse(exception.message[6..-1]).to_h.dig("error", "root_cause", 0, "reason") + + render json: { "errors" => { "title" => message }}.to_json, status: :bad_request + end + end + + def organizations + begin + + # Loop through all providers + providers = [] + + page = { size: 1000, number: 1} + response = Provider.query(nil, page: page, include_deleted: true) + providers = providers + response.records.to_a + + total = response.results.total + total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 + + # keep going for all pages + page_num = 2 + while page_num <= total_pages + page = { size: 1000, number: page_num } + response = Provider.query(nil, page: page) + providers = providers + response.records.to_a + page_num += 1 + end + + # Loop through all clients + clients = [] + + page = { size: 1000, number: 1} + response = Client.query(nil, page: page, include_deleted: true) + clients = clients + response.records.to_a + + total = response.results.total + total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 + + # keep going for all pages + page_num = 2 + while page_num <= total_pages + page = { size: 1000, number: page_num } + response = Client.query(nil, page: page) + clients = clients + response.records.to_a + page_num += 1 + end + + # Get doi counts via DOIS query and combine next to clients. + response = Doi.query(nil, state: "registered,findable", page: { size: 0, number: 1}, totals_agg: true) + + client_totals = {} + totals_buckets = response.response.aggregations.clients_totals.buckets + totals_buckets.each do |totals| + client_totals[totals["key"]] = { + "count" => totals["doc_count"], + "this_year" => totals.this_year.buckets[0]["doc_count"], + "last_year" => totals.last_year.buckets[0]["doc_count"] + } + end + + respond_to do |format| + format.csv do + headers = %W( + accountName + fabricaAccountId + parentFabricaAccountId + salesForceId + parentSalesForceId + isActive + accountDescription + accountWebsite + region + focusArea + organisationType + accountType + generalContactEmail + groupEmail + billingStreet + billingPostalCode + billingCity + billingDepartment + billingOrganization + billingState + billingCountry + twitter + rorId + created + deleted + doisCountCurrentYear + doisCountPreviousYear + doisCountTotal + ) + + csv = headers.to_csv + + providers.each do |provider| + row = { + accountName: provider.name, + fabricaAccountId: provider.symbol, + parentFabricaAccountId: provider.consortium.present? ? provider.consortium.symbol : nil, + salesForceId: provider.salesforce_id, + parentSalesForceId: provider.consortium.present? ? provider.consortium.salesforce_id : nil, + isActive: provider.is_active == "\x01", + accountDescription: provider.description, + accountWebsite: provider.website, + region: provider.region_human_name, + focusArea: provider.focus_area, + organizationType: provider.organization_type, + accountType: provider.member_type_label, + generalContactEmail: provider.system_email, + groupEmail: provider.group_email, + billingStreet: provider.billing_address, + billingPostalCode: provider.billing_post_code, + billingCity: provider.billing_city, + billingDepartment: provider.billing_department, + billingOrganization: provider.billing_organization, + billingState: provider.billing_state, + billingCountry: provider.billing_country, + twitter: provider.twitter_handle, + rorId: provider.ror_id, + created: provider.created, + deleted: provider.deleted_at, + doisCountCurrentYear: nil, + doisCountPreviousYear: nil, + doisCountTotal: nil + }.values + + csv += CSV.generate_line row + end + + clients.each do |client| + client_id = client.symbol.downcase + row = { + accountName: client.name, + fabricaAccountId: client.symbol, + parentFabricaAccountId: client.provider.present? ? client.provider.symbol : nil, + salesForceId: client.salesforce_id, + parentSalesForceId: client.provider.present? ? client.provider.salesforce_id : nil, + isActive: client.is_active == "\x01", + accountDescription: client.description, + accountWebsite: client.url, + region: nil, + focusArea: nil, + organisztionType: nil, + accountType: 'repository', + generalContactEmail: client.system_email, + groupEmail: nil, + billingStreet: nil, + billingPostalCode: nil, + billingCity: nil, + billingDepartment: nil, + billingOrganization: nil, + billingState: nil, + billingCountry: nil, + twitter: nil, + rorId: nil, + created: client.created, + deleted: client.deleted_at, + doisCountCurrentYear: client_totals[client_id] ? client_totals[client_id]["this_year"] : nil, + doisCountPreviousYear: client_totals[client_id] ? client_totals[client_id]["last_year"] : nil, + doisCountTotal: client_totals[client_id] ? client_totals[client_id]["count"] : nil + }.values + + csv += CSV.generate_line row + end + + send_data csv, filename: "organizations-#{Date.today}.csv" + end + end + + rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception + Raven.capture_exception(exception) + + message = JSON.parse(exception.message[6..-1]).to_h.dig("error", "root_cause", 0, "reason") + + render json: { "errors" => { "title" => message }}.to_json, status: :bad_request + end + end + +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index ffad89686..92dad0ecc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -57,6 +57,11 @@ get 'clients/totals', :to => 'clients#totals' get 'prefixes/totals', :to => 'prefixes#totals' + # Reporting + get 'reports/organizations', :to => 'reports#organizations', defaults: { format: :csv } + get 'reports/contacts', :to => 'reports#contacts', defaults: { format: :csv } + + resources :heartbeat, only: [:index] resources :index, only: [:index] From 78bf865e1b4bb0d6a755680bc06b251603a5051f Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Mon, 30 Sep 2019 17:23:23 +0200 Subject: [PATCH 2/8] Restrict reports --- app/controllers/reports_controller.rb | 6 +++--- app/models/ability.rb | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index 5b968999f..746d5d7a7 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -1,10 +1,10 @@ class ReportsController < ApplicationController include ActionController::MimeResponds - def contacts - begin + authorize! :read, :reports + begin # Loop through all providers providers = [] @@ -104,8 +104,8 @@ def contacts end def organizations + authorize! :read, :reports begin - # Loop through all providers providers = [] diff --git a/app/models/ability.rb b/app/models/ability.rb index f40f1371b..f39caca0d 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -14,6 +14,7 @@ def initialize(user) cannot [:new, :create], Doi do |doi| doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first)) end + can :read, :reports elsif user.role_id == "staff_user" can :read, :all elsif user.role_id == "provider_admin" && user.provider_id.present? @@ -66,7 +67,7 @@ def initialize(user) # end can [:read, :destroy, :update, :register_url, :validate, :undo, :get_url, :get_urls, :read_landing_page_results], Doi, :client_id => user.client_id - can [:new, :create], Doi do |doi| + can [:new, :create], Doi do |doi| doi.client.prefixes.where(prefix: doi.prefix).present? || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first) end can [:read], Doi do |doi| From 2751e1db4c9a1657df531370a0a2f583f5aca307 Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Tue, 1 Oct 2019 10:03:02 +0200 Subject: [PATCH 3/8] Rename reports for exports to avoid confusion --- .../{reports_controller.rb => export_controller.rb} | 6 +++--- app/models/ability.rb | 3 ++- config/routes.rb | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) rename app/controllers/{reports_controller.rb => export_controller.rb} (98%) diff --git a/app/controllers/reports_controller.rb b/app/controllers/export_controller.rb similarity index 98% rename from app/controllers/reports_controller.rb rename to app/controllers/export_controller.rb index 746d5d7a7..c1ecdfb27 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/export_controller.rb @@ -1,8 +1,8 @@ -class ReportsController < ApplicationController +class ExportController < ApplicationController include ActionController::MimeResponds def contacts - authorize! :read, :reports + authorize! :export, :contacts begin # Loop through all providers @@ -104,7 +104,7 @@ def contacts end def organizations - authorize! :read, :reports + authorize! :export, :organizations begin # Loop through all providers providers = [] diff --git a/app/models/ability.rb b/app/models/ability.rb index d41359779..542a55aa8 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -14,7 +14,8 @@ def initialize(user) cannot [:new, :create], Doi do |doi| doi.client.blank? || !(doi.client.prefixes.where(prefix: doi.prefix).first || %w(crossref medra kisti jalc op).include?(doi.client.symbol.downcase.split(".").first)) end - can :read, :reports + can :export, :contacts + can :export, :organizations elsif user.role_id == "staff_user" can :read, :all elsif user.role_id == "provider_admin" && user.provider_id.present? diff --git a/config/routes.rb b/config/routes.rb index 69b65d25b..ca9a6d55a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -58,8 +58,8 @@ get 'prefixes/totals', :to => 'prefixes#totals' # Reporting - get 'reports/organizations', :to => 'reports#organizations', defaults: { format: :csv } - get 'reports/contacts', :to => 'reports#contacts', defaults: { format: :csv } + get 'export/organizations', :to => 'export#organizations', defaults: { format: :csv } + get 'export/contacts', :to => 'export#contacts', defaults: { format: :csv } resources :heartbeat, only: [:index] From 74b36d70505e0ebf132d75bd9f1b254ec60b4590 Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Tue, 1 Oct 2019 14:09:22 +0200 Subject: [PATCH 4/8] Auth before for export --- app/controllers/export_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index c1ecdfb27..8552703b9 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -1,6 +1,8 @@ class ExportController < ApplicationController include ActionController::MimeResponds + before_action :authenticate_user! + def contacts authorize! :export, :contacts From d0755756adaf8e1b34d99a7d46d7f523c4ebe61c Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Wed, 2 Oct 2019 09:09:25 +0200 Subject: [PATCH 5/8] Basic auth prompt for exports --- app/controllers/application_controller.rb | 16 ++++++++++++++-- app/controllers/export_controller.rb | 2 +- config/application.rb | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 7252fef37..770326092 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,4 +1,5 @@ class ApplicationController < ActionController::API + include ActionController::HttpAuthentication::Basic::ControllerMethods include Authenticable include CanCan::ControllerAdditions include ErrorSerializable @@ -52,6 +53,17 @@ def default_format_json request.format = :json if request.format.html? end + def authenticate_user_with_basic_auth! + @user = authenticate_user! + + if !@user + request_http_basic_authentication(realm = ENV['REALM']) + end + puts @user.role_id + + @user + end + def authenticate_user! type, credentials = type_and_credentials_from_request_headers return false unless credentials.present? @@ -83,7 +95,7 @@ def authenticated_user when "ActionController::UnknownFormat" then 406 when "ActiveRecord::RecordNotUnique" then 409 when "ActiveModel::ForbiddenAttributesError", "ActionController::ParameterMissing", "ActionController::UnpermittedParameters", "ActiveModelSerializers::Adapter::JsonApi::Deserialization::InvalidDocument" then 422 - when "SocketError" then 500 + when "SocketError" then 500 else 400 end @@ -135,7 +147,7 @@ def set_raven_context else Raven.user_context( ip_address: request.ip - ) + ) end end end diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index 8552703b9..49f41ed1f 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -1,7 +1,7 @@ class ExportController < ApplicationController include ActionController::MimeResponds - before_action :authenticate_user! + before_action :authenticate_user_with_basic_auth! def contacts authorize! :export, :contacts diff --git a/config/application.rb b/config/application.rb index 516a1dbfc..3c35f52e2 100644 --- a/config/application.rb +++ b/config/application.rb @@ -56,6 +56,7 @@ ENV['MG_FROM'] ||= "support@datacite.org" ENV['MG_DOMAIN'] ||= "mg.datacite.org" ENV['HANDLES_MINTED'] ||= "10132" +ENV['REALM'] ||= ENV['API_URL'] module Lupo class Application < Rails::Application From 2e1aa215a4f8feb951d711b52e9b6550235abbd9 Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Wed, 2 Oct 2019 10:15:54 +0200 Subject: [PATCH 6/8] Remove accidental debug code --- app/controllers/application_controller.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 770326092..411e2f9b6 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -59,7 +59,6 @@ def authenticate_user_with_basic_auth! if !@user request_http_basic_authentication(realm = ENV['REALM']) end - puts @user.role_id @user end From 33405b2cca0c776554d2eee1c89c4937f0dd28ea Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Wed, 2 Oct 2019 16:55:12 +0200 Subject: [PATCH 7/8] enable filtering out crossref.citations --- app/controllers/dois_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/dois_controller.rb b/app/controllers/dois_controller.rb index 2bd08c2d3..f5a758c5a 100644 --- a/app/controllers/dois_controller.rb +++ b/app/controllers/dois_controller.rb @@ -53,6 +53,7 @@ def index else response = Doi.query(params[:query], state: params[:state], + exclude_registration_agencies: params[:exclude_registration_agencies], created: params[:created], registered: params[:registered], provider_id: params[:provider_id], From 042dbb2a459ae60b6ce51b200f380267d695fb83 Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Thu, 3 Oct 2019 16:52:54 +0200 Subject: [PATCH 8/8] Remove commas from headers in contacts --- app/controllers/export_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index 49f41ed1f..3c8c85660 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -30,9 +30,9 @@ def contacts format.csv do headers = %W( fabricaAccountId - email, - firstName, - lastName, + email + firstName + lastName type )