From 8e680d8014ccb81fbeee823ffddc8a2716a7dbee Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Sun, 8 Mar 2020 07:40:46 +0100 Subject: [PATCH] specs for csv export --- app/controllers/export_controller.rb | 361 +++++++++++++-------------- app/models/concerns/indexable.rb | 2 +- spec/factories/default.rb | 8 +- spec/models/provider_spec.rb | 15 ++ spec/requests/exports_spec.rb | 53 ++-- 5 files changed, 223 insertions(+), 216 deletions(-) diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index 244d8607b..8893cebc4 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -10,11 +10,9 @@ def contacts begin # Loop through all providers - providers = [] - page = { size: 1000, number: 1} response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) - providers = providers + response.records.to_a + providers = response.results.to_a total = response.results.total total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 @@ -24,69 +22,65 @@ def contacts while page_num <= total_pages page = { size: 1000, number: page_num } response = Provider.query(nil, page: page) - providers = providers + response.records.to_a + providers = providers + response.results.to_a page_num += 1 end - respond_to do |format| - format.csv do - headers = %W( - fabricaAccountId - fabricaId - email - firstName - lastName - type - ) - - csv = headers.to_csv - - # Use a hashmap for the contacts to avoid duplicated - contacts = Hash.new - - add_contact = Proc.new { |contacts, email, id, firstname, lastname, type| - if email - unless contacts.has_key?(email) - contacts[email] = { - 'fabricaAccountId' => id, - 'fabricaId' => id + "-" + email, - 'firstName' => firstname, - 'lastName' => lastname.present? ? lastname : email, - } - end - - if contacts[email].has_key?('type') - contacts[email]['type'] += ";" + type - else - contacts[email]['type'] = type - end - end - } - - providers.each do |provider| - add_contact.call(contacts, provider.technical_contact_email, provider.symbol, provider.technical_contact_given_name, provider.technical_contact_family_name, 'technical') - add_contact.call(contacts, provider.secondary_technical_contact_email, provider.symbol, provider.secondary_technical_contact_given_name, provider.secondary_technical_contact_family_name, 'secondaryTechnical') - add_contact.call(contacts, provider.service_contact_email, provider.symbol, provider.service_contact_given_name, provider.service_contact_family_name, 'service') - add_contact.call(contacts, provider.secondary_service_contact_email, provider.symbol, provider.secondary_service_contact_given_name, provider.secondary_service_contact_family_name, 'secondaryService') - add_contact.call(contacts, provider.voting_contact_email, provider.symbol, provider.voting_contact_given_name, provider.voting_contact_family_name, 'voting') - add_contact.call(contacts, provider.billing_contact_email, provider.symbol, provider.billing_contact_given_name, provider.billing_contact_family_name, 'billing') - add_contact.call(contacts, provider.secondary_billing_contact_email, provider.symbol, provider.secondary_billing_contact_given_name, provider.secondary_billing_contact_family_name, 'secondaryBilling') + headers = %W( + fabricaAccountId + fabricaId + email + firstName + lastName + type + ) + + csv = headers.to_csv + + # Use a hashmap for the contacts to avoid duplicated + contacts = Hash.new + + add_contact = Proc.new { |contacts, email, id, firstname, lastname, type| + if email + unless contacts.has_key?(email) + contacts[email] = { + 'fabricaAccountId' => id, + 'fabricaId' => id + "-" + email, + 'firstName' => firstname, + 'lastName' => lastname.present? ? lastname : email, + } end - contacts.each do |email, contact| - csv += CSV.generate_line [ - contact['fabricaAccountId'], - contact['fabricaId'], - email, - contact['firstName'], - contact['lastName'], - contact['type'], - ] + if contacts[email].has_key?('type') + contacts[email]['type'] += ";" + type + else + contacts[email]['type'] = type end - - send_data csv, filename: "contacts-#{Date.today}.csv" end + } + + providers.each do |provider| + add_contact.call(contacts, provider.technical_contact.email, provider.symbol, provider.technical_contact.given_name, provider.technical_contact.family_name, 'technical') + add_contact.call(contacts, provider.secondary_technical_contact.email, provider.symbol, provider.secondary_technical_contact.given_name, provider.secondary_technical_contact.family_name, 'secondaryTechnical') + add_contact.call(contacts, provider.service_contact.email, provider.symbol, provider.service_contact.given_name, provider.service_contact.family_name, 'service') + add_contact.call(contacts, provider.secondary_service_contact.email, provider.symbol, provider.secondary_service_contact.given_name, provider.secondary_service_contact.family_name, 'secondaryService') + add_contact.call(contacts, provider.voting_contact.email, provider.symbol, provider.voting_contact.given_name, provider.voting_contact.family_name, 'voting') + add_contact.call(contacts, provider.billing_contact.email, provider.symbol, provider.billing_contact.given_name, provider.billing_contact.family_name, 'billing') + add_contact.call(contacts, provider.secondary_billing_contact.email, provider.symbol, provider.secondary_billing_contact.given_name, provider.secondary_billing_contact.family_name, 'secondaryBilling') end + + contacts.each do |email, contact| + csv += CSV.generate_line [ + contact['fabricaAccountId'], + contact['fabricaId'], + email, + contact['firstName'], + contact['lastName'], + contact['type'], + ] + end + + send_data csv, filename: "contacts-#{Date.today}.csv" rescue StandardError, Elasticsearch::Transport::Transport::Errors::BadRequest => exception Raven.capture_exception(exception) @@ -98,13 +92,12 @@ def contacts def organizations authorize! :export, :organizations + begin # Loop through all providers - providers = [] - - page = { size: 1000, number: 1} + page = { size: 1000, number: 1 } response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) - providers = providers + response.records.to_a + providers = response.results.to_a total = response.results.total total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 @@ -113,76 +106,72 @@ def organizations page_num = 2 while page_num <= total_pages page = { size: 1000, number: page_num } - response = Provider.query(nil, page: page) - providers = providers + response.records.to_a + response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + providers = providers + response.results.to_a page_num += 1 end - respond_to do |format| - format.csv do - headers = [ - "Name", - "fabricaAccountId", - "Parent Organization", - "Is Active", - "Organization Description", - "Website", - "Region", - "Focus Area", - "Sector", - "Member Type", - "Email", - "Group Email", - "billingStreet", - "Billing Zip/Postal Code", - "billingCity", - "Department", - "billingOrganization", - "billingStateCode", - "billingCountryCode", - "twitter", - "ROR", - "Fabrica Creation Date", - "Fabrica Modification Date", - "Fabrica Deletion Date" - ] - - csv = headers.to_csv - - providers.each do |provider| - row = { - accountName: provider.name, - fabricaAccountId: provider.symbol, - parentFabricaAccountId: provider.consortium.present? ? provider.consortium.symbol : nil, - isActive: provider.deleted_at.blank?, - accountDescription: provider.description, - accountWebsite: provider.website, - region: provider.region_human_name, - focusArea: provider.focus_area, - sector: provider.organization_type, - accountType: provider.member_type_label, - generalContactEmail: provider.system_email, - groupEmail: provider.group_email, - billingStreet: provider.billing_address, - billingPostalCode: provider.billing_post_code, - billingCity: provider.billing_city, - billingDepartment: provider.billing_department, - billingOrganization: provider.billing_organization, - billingStateCode: provider.billing_state.present? ? provider.billing_state.split("-").last : nil, - billingCountryCode: provider.billing_country, - twitter: provider.twitter_handle, - rorId: provider.ror_id, - created: provider.created.strftime(EXPORT_DATE_FORMAT), - modified: provider.updated.strftime(EXPORT_DATE_FORMAT), - deleted: provider.deleted_at.present? ? provider.deleted_at.strftime(EXPORT_DATE_FORMAT) : nil, - }.values - - csv += CSV.generate_line row - end - - send_data csv, filename: "organizations-#{Date.today}.csv" - end + headers = [ + "Name", + "fabricaAccountId", + "Parent Organization", + "Is Active", + "Organization Description", + "Website", + "Region", + "Focus Area", + "Sector", + "Member Type", + "Email", + "Group Email", + "billingStreet", + "Billing Zip/Postal Code", + "billingCity", + "Department", + "billingOrganization", + "billingStateCode", + "billingCountryCode", + "twitter", + "ROR", + "Fabrica Creation Date", + "Fabrica Modification Date", + "Fabrica Deletion Date" + ] + + csv = headers.to_csv + + providers.each do |provider| + row = { + accountName: provider.name, + fabricaAccountId: provider.symbol, + parentFabricaAccountId: provider.consortium_id.present? ? provider.consortium_id.upcase : nil, + isActive: provider.deleted_at.blank?, + accountDescription: provider.description, + accountWebsite: provider.website, + region: provider.region, + focusArea: provider.focus_area, + sector: provider.organization_type, + accountType: provider.member_type, + generalContactEmail: provider.system_email, + groupEmail: provider.group_email, + billingStreet: provider.billing_information.address, + billingPostalCode: provider.billing_information.post_code, + billingCity: provider.billing_information.city, + billingDepartment: provider.billing_information.department, + billingOrganization: provider.billing_information.organization, + billingStateCode: provider.billing_information.state.present? ? provider.billing_information.state.split("-").last : nil, + billingCountryCode: provider.billing_information.country, + twitter: provider.twitter_handle, + rorId: provider.ror_id, + created: provider.created, + modified: provider.updated, + deleted: provider.deleted_at.present? ? provider.deleted_at : nil, + }.values + + csv += CSV.generate_line row end + + send_data csv, filename: "organizations-#{Date.today}.csv" rescue StandardError, Elasticsearch::Transport::Transport::Errors::BadRequest => exception Raven.capture_exception(exception) @@ -196,11 +185,9 @@ def repositories #authorize! :export, :repositories begin # Loop through all clients - clients = [] - - page = { size: 1000, number: 1} + page = { size: 1000, number: 1 } response = Client.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) - clients = clients + response.records.to_a + clients = response.results.to_a total = response.results.total total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 @@ -209,13 +196,13 @@ def repositories page_num = 2 while page_num <= total_pages page = { size: 1000, number: page_num } - response = Client.query(nil, page: page, include_deleted: true) - clients = clients + response.records.to_a + response = Client.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + clients = clients + response.results.to_a page_num += 1 end # Get doi counts via DOIs query and combine next to clients. - response = Doi.query(nil, state: "registered,findable", page: { size: 0, number: 1}, totals_agg: "client") + response = Doi.query(nil, state: "registered,findable", page: { size: 0, number: 1 }, totals_agg: "client") client_totals = {} totals_buckets = response.response.aggregations.clients_totals.buckets @@ -227,62 +214,56 @@ def repositories } end - respond_to do |format| - format.csv do - headers = [ - "Repository Name", - "Repository ID", - "Organization", - "isActive", - "Description", - "Repository URL", - "generalContactEmail", - "serviceContactEmail", - "serviceContactGivenName", - "serviceContactFamilyName", - "Fabrica Creation Date", - "Fabrica Modification Date", - "Fabrica Deletion Date", - "doisCurrentYear", - "doisPreviousYear", - "doisTotal" - ] - - csv = headers.to_csv - - clients.each do |client| - client_id = client.symbol.downcase - - # Limit for salesforce default of max 80 chars - name = +client.name.truncate(80) - # Clean the name to remove quotes, which can break csv parsers - name.gsub! /["']/, '' - - row = { - accountName: name, - fabricaAccountId: client.symbol, - parentFabricaAccountId: client.provider.present? ? client.provider.symbol : nil, - isActive: client.deleted_at.blank?, - accountDescription: client.description, - accountWebsite: client.url, - generalContactEmail: client.system_email, - serviceContactEmail: client.service_contact_email, - serviceContactGivenName: client.service_contact_given_name, - serviceContactFamilyName: client.service_contact_family_name, - created: client.created.strftime(EXPORT_DATE_FORMAT), - modified: client.updated.strftime(EXPORT_DATE_FORMAT), - deleted: client.deleted_at.present? ? client.deleted_at.strftime(EXPORT_DATE_FORMAT) : nil, - doisCountCurrentYear: client_totals[client_id] ? client_totals[client_id]["this_year"] : nil, - doisCountPreviousYear: client_totals[client_id] ? client_totals[client_id]["last_year"] : nil, - doisCountTotal: client_totals[client_id] ? client_totals[client_id]["count"] : nil - }.values - - csv += CSV.generate_line row - end - - send_data csv, filename: "repositories-#{Date.today}.csv" - end + headers = [ + "Repository Name", + "Repository ID", + "Organization", + "isActive", + "Description", + "Repository URL", + "generalContactEmail", + "serviceContactEmail", + "serviceContactGivenName", + "serviceContactFamilyName", + "Fabrica Creation Date", + "Fabrica Modification Date", + "Fabrica Deletion Date", + "doisCurrentYear", + "doisPreviousYear", + "doisTotal" + ] + + csv = headers.to_csv + + clients.each do |client| + # Limit for salesforce default of max 80 chars + name = +client.name.truncate(80) + # Clean the name to remove quotes, which can break csv parsers + name.gsub! /["']/, '' + + row = { + accountName: name, + fabricaAccountId: client.symbol, + parentFabricaAccountId: client.provider.present? ? client.provider.symbol : nil, + isActive: client.deleted_at.blank?, + accountDescription: client.description, + accountWebsite: client.url, + generalContactEmail: client.system_email, + serviceContactEmail: client.service_contact.email, + serviceContactGivenName: client.service_contact.given_name, + serviceContactFamilyName: client.service_contact.family_name, + created: client.created, + modified: client.updated, + deleted: client.deleted_at.present? ? client.deleted_at : nil, + doisCountCurrentYear: client_totals[client.uid] ? client_totals[client.uid]["this_year"] : nil, + doisCountPreviousYear: client_totals[client.uid] ? client_totals[client.uid]["last_year"] : nil, + doisCountTotal: client_totals[client.uid] ? client_totals[client.uid]["count"] : nil + }.values + + csv += CSV.generate_line row end + + send_data csv, filename: "repositories-#{Date.today}.csv" rescue StandardError, Elasticsearch::Transport::Transport::Errors::BadRequest => exception Raven.capture_exception(exception) diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index ada8c056d..61e3f9251 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -219,7 +219,7 @@ def query(query, options={}) must_not << { exists: { field: "deleted_at" }} unless options[:include_deleted] if options[:exclude_registration_agencies] - must_not << { terms: { role_name: ["ROLE_ADMIN", "ROLE_REGISTRATION_AGENCY", "ROLE_CONSORTIUM_ORGANIZATION"] }} + must_not << { terms: { role_name: ["ROLE_ADMIN", "ROLE_REGISTRATION_AGENCY"] }} else must_not << { term: { role_name: "ROLE_ADMIN" }} end diff --git a/spec/factories/default.rb b/spec/factories/default.rb index e8063a818..0409f4f6e 100644 --- a/spec/factories/default.rb +++ b/spec/factories/default.rb @@ -276,14 +276,14 @@ "family_name": "Garza" }} billing_contact {{ - "email": "Trisha@example.com", + "email": "trisha@example.com", "given_name": "Trisha", - "family_name": "cruse" + "family_name": "Cruse" }} secondary_billing_contact {{ - "email": "Trisha@example.com", + "email": "trisha@example.com", "given_name": "Trisha", - "family_name": "cruse" + "family_name": "Cruse" }} service_contact {{ "email": "martin@example.com", diff --git a/spec/models/provider_spec.rb b/spec/models/provider_spec.rb index f4574acc8..5cb1d66bc 100644 --- a/spec/models/provider_spec.rb +++ b/spec/models/provider_spec.rb @@ -134,6 +134,21 @@ expect(consortium_organization.member_type).to eq("consortium_organization") end end + + describe "provider with ROLE_CONSORTIUM_ORGANIZATION" do + let(:consortium) { create(:provider, role_name: "ROLE_CONSORTIUM", name: "Virtual Library of Virginia", symbol: "VIVA") } + + subject { create(:provider, name: "University of Virginia", role_name: "ROLE_CONSORTIUM_ORGANIZATION", consortium_id: consortium.symbol) } + + it "works" do + expect(subject.role_name).to eq("ROLE_CONSORTIUM_ORGANIZATION") + expect(subject.member_type).to eq("consortium_organization") + expect(subject.member_type_label).to eq("Consortium Organization") + expect(subject.consortium.name).to eq("Virtual Library of Virginia") + expect(subject.consortium.symbol).to eq("VIVA") + expect(subject.consortium.member_type).to eq("consortium") + end + end describe "to_jsonapi" do it "works" do diff --git a/spec/requests/exports_spec.rb b/spec/requests/exports_spec.rb index e698a96c0..ed19526fc 100644 --- a/spec/requests/exports_spec.rb +++ b/spec/requests/exports_spec.rb @@ -1,59 +1,70 @@ require "rails_helper" describe "exports", type: :request do - let(:admin) { create(:provider, symbol: "ADMIN") } - let(:admin_bearer) { Client.generate_token(role_id: "staff_admin", uid: admin.symbol, password: admin.password) } - let(:admin_headers) { { "HTTP_ACCEPT" => "application/vnd.api+json", "HTTP_AUTHORIZATION" => "Bearer " + admin_bearer} } - - let(:provider) { create(:provider, symbol: "DATACITE") } - let(:client) { create(:client, provider: provider, symbol: ENV['MDS_USERNAME'], password: ENV['MDS_PASSWORD']) } - let!(:prefix) { create(:prefix, prefix: "10.14454") } - let!(:client_prefix) { create(:client_prefix, client: client, prefix: prefix) } - - let(:doi) { create(:doi, client: client) } - let(:headers) { { "HTTP_ACCEPT" => "application/vnd.api+json", "HTTP_AUTHORIZATION" => "Bearer " + admin_bearer }} - let!(:dois) { create_list(:doi, 3, client: client, aasm_state: "findable") } + let(:admin_bearer) { User.generate_token } + let(:admin_headers) { { "HTTP_ACCEPT" => "text/csv", "HTTP_AUTHORIZATION" => "Bearer " + admin_bearer} } + let(:consortium) { create(:provider, role_name: "ROLE_CONSORTIUM", name: "Virtual Library of Virginia", symbol: "VIVA") } + let!(:provider) { create(:provider, role_name: "ROLE_CONSORTIUM_ORGANIZATION", name: "University of Virginia", symbol: "UVA", consortium: consortium) } + describe "GET /export/organizations", elasticsearch: true do before do - Doi.import Provider.import - sleep 2 + sleep 1 end it 'returns organizations', vcr: false do - get "/export/organizations", nil, headers + get "/export/organizations", nil, admin_headers expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(3) + expect(csv[0]).to start_with("Name,fabricaAccountId,Parent Organization,Is Active") + expect(csv[1]).to start_with("Virtual Library of Virginia,VIVA,,true") + expect(csv[2]).to start_with("University of Virginia,UVA,VIVA,true") end end describe "GET /export/repositories", elasticsearch: true do + let(:client) { create(:client, provider: provider, symbol: "UVA.LIBRARY", name: "University of Virginia Library") } + let!(:dois) { create_list(:doi, 3, client: client, aasm_state: "findable") } + before do Doi.import Client.import - sleep 2 + sleep 1 end it 'returns repositories', vcr: false do - get "/export/repositories", nil, headers + get "/export/repositories", nil, admin_headers expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(2) + expect(csv[0]).to start_with("Repository Name,Repository ID,Organization,isActive") + expect(csv[1]).to start_with("University of Virginia Library,UVA.LIBRARY,UVA,true") + dois_total = csv[1].strip.split(",").last.to_i + expect(dois_total).to eq(3) end end describe "GET /export/contacts", elasticsearch: true do before do - Doi.import - Client.import Provider.import - sleep 2 + sleep 1 end it 'returns contacts', vcr: false do - get "/export/contacts", nil, headers + get "/export/contacts", nil, admin_headers expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(5) + expect(csv[0]).to eq("fabricaAccountId,fabricaId,email,firstName,lastName,type\n") + expect(csv[1]).to start_with("VIVA,VIVA-kristian@example.com,kristian@example.com,Kristian,Garza,technical;secondaryTechnical") + expect(csv[2]).to start_with("VIVA,VIVA-martin@example.com,martin@example.com,Martin,Fenner,service;secondaryService") + expect(csv[3]).to start_with("VIVA,VIVA-robin@example.com,robin@example.com,Robin,Dasler,voting") + expect(csv[4]).to start_with("VIVA,VIVA-trisha@example.com,trisha@example.com,Trisha,Cruse,billing;secondaryBilling") end end end