From 6d1c13ead19aeb3121d8513da580fc36f8ff96c7 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Mon, 9 Mar 2020 11:36:11 +0100 Subject: [PATCH] support from-date parameter in csv export --- app/controllers/clients_controller.rb | 3 +- app/controllers/export_controller.rb | 12 ++++---- app/controllers/providers_controller.rb | 3 +- app/controllers/repositories_controller.rb | 1 + app/models/concerns/indexable.rb | 2 ++ spec/requests/exports_spec.rb | 36 ++++++++++++++++++++++ 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index 173f2f4b7..2ae3a2edd 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -24,7 +24,8 @@ def index response = Client.find_by_id(params[:ids], page: page, sort: sort) else response = Client.query(params[:query], - year: params[:year], + year: params[:year], + from_date: params[:from_date], provider_id: params[:provider_id], re3data_id: params[:re3data_id], opendoar_id: params[:opendoar_id], diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index 6e0852529..815248383 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -24,7 +24,7 @@ def contacts begin # Loop through all providers page = { size: 1000, number: 1} - response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + response = Provider.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) providers = response.results.to_a total = response.results.total @@ -34,7 +34,7 @@ def contacts page_num = 2 while page_num <= total_pages page = { size: 1000, number: page_num } - response = Provider.query(nil, page: page) + response = Provider.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) providers = providers + response.results.to_a page_num += 1 end @@ -117,7 +117,7 @@ def organizations begin # Loop through all providers page = { size: 1000, number: 1 } - response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + response = Provider.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) providers = response.results.to_a total = response.results.total @@ -127,7 +127,7 @@ def organizations page_num = 2 while page_num <= total_pages page = { size: 1000, number: page_num } - response = Provider.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + response = Provider.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) providers = providers + response.results.to_a page_num += 1 end @@ -207,7 +207,7 @@ def repositories begin # Loop through all clients page = { size: 1000, number: 1 } - response = Client.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + response = Client.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) clients = response.results.to_a total = response.results.total @@ -217,7 +217,7 @@ def repositories page_num = 2 while page_num <= total_pages page = { size: 1000, number: page_num } - response = Client.query(nil, page: page, include_deleted: true, exclude_registration_agencies: true) + response = Client.query(nil, page: page, from_date: params[:from_date], include_deleted: true, exclude_registration_agencies: true) clients = clients + response.results.to_a page_num += 1 end diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index bcefa4996..3e707de6e 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -26,7 +26,8 @@ def index else response = Provider.query(params[:query], exclude_registration_agencies: params[:exclude_registration_agencies], - year: params[:year], + year: params[:year], + from_date: params[:from_date], region: params[:region], consortium_id: params[:consortium_id], member_type: params[:member_type], diff --git a/app/controllers/repositories_controller.rb b/app/controllers/repositories_controller.rb index 2297e467d..60a14e12a 100644 --- a/app/controllers/repositories_controller.rb +++ b/app/controllers/repositories_controller.rb @@ -26,6 +26,7 @@ def index else response = Client.query(params[:query], year: params[:year], + from_date: params[:from_date], provider_id: params[:provider_id], consortium_id: params[:consortium_id], re3data_id: params[:re3data_id], diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index 61e3f9251..45f13763e 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -210,6 +210,7 @@ def query(query, options={}) # filters for some classes if self.name == "Provider" must << { range: { created: { gte: "#{options[:year].split(",").min}||/y", lte: "#{options[:year].split(",").max}||/y", format: "yyyy" }}} if options[:year].present? + must << { range: { updated: { gte: "#{options[:from_date]}||/d" }}} if options[:from_date].present? must << { term: { region: options[:region].upcase }} if options[:region].present? must << { term: { "consortium_id.raw" => options[:consortium_id] }} if options[:consortium_id].present? must << { term: { member_type: options[:member_type] }} if options[:member_type].present? @@ -225,6 +226,7 @@ def query(query, options={}) end elsif self.name == "Client" must << { range: { created: { gte: "#{options[:year].split(",").min}||/y", lte: "#{options[:year].split(",").max}||/y", format: "yyyy" }}} if options[:year].present? + must << { range: { updated: { gte: "#{options[:from_date]}||/d" }}} if options[:from_date].present? must << { terms: { "software.raw" => options[:software].split(",") }} if options[:software].present? must << { terms: { certificate: options[:certificate].split(",") }} if options[:certificate].present? must << { terms: { repository_type: options[:repository_type].split(",") }} if options[:repository_type].present? diff --git a/spec/requests/exports_spec.rb b/spec/requests/exports_spec.rb index 11e4df5bb..e09f378a9 100644 --- a/spec/requests/exports_spec.rb +++ b/spec/requests/exports_spec.rb @@ -23,6 +23,17 @@ expect(csv[1]).to start_with("Virtual Library of Virginia,VIVA,,true") expect(csv[2]).to start_with("University of Virginia,UVA,VIVA,true") end + + it 'returns organizations from date', vcr: false do + get "/export/organizations?from-date=#{Date.today}", nil, admin_headers + + expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(3) + expect(csv[0]).to start_with("Name,fabricaAccountId,Parent Organization,Is Active") + expect(csv[1]).to start_with("Virtual Library of Virginia,VIVA,,true") + expect(csv[2]).to start_with("University of Virginia,UVA,VIVA,true") + end end describe "GET /export/repositories", elasticsearch: true do @@ -46,6 +57,18 @@ dois_total = csv[1].strip.split(",").last.to_i expect(dois_total).to eq(3) end + + it 'returns repositories from date', vcr: false do + get "/export/repositories?from-date=#{Date.today}", nil, admin_headers + + expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(2) + expect(csv[0]).to start_with("Repository Name,Repository ID,Organization,isActive") + expect(csv[1]).to start_with("University of Virginia Library,UVA.LIBRARY,UVA,true") + dois_total = csv[1].strip.split(",").last.to_i + expect(dois_total).to eq(3) + end end describe "GET /export/contacts", elasticsearch: true do @@ -67,6 +90,19 @@ expect(csv[4]).to start_with("VIVA,VIVA-trisha@example.com,trisha@example.com,Trisha,Cruse,billing;secondaryBilling") end + it 'returns all contacts from date', vcr: false do + get "/export/contacts?from-date=#{Date.today}", nil, admin_headers + + expect(last_response.status).to eq(200) + csv = last_response.body.lines + expect(csv.length).to eq(5) + expect(csv[0]).to eq("fabricaAccountId,fabricaId,email,firstName,lastName,type\n") + expect(csv[1]).to start_with("VIVA,VIVA-kristian@example.com,kristian@example.com,Kristian,Garza,technical;secondaryTechnical") + expect(csv[2]).to start_with("VIVA,VIVA-martin@example.com,martin@example.com,Martin,Fenner,service;secondaryService") + expect(csv[3]).to start_with("VIVA,VIVA-robin@example.com,robin@example.com,Robin,Dasler,voting") + expect(csv[4]).to start_with("VIVA,VIVA-trisha@example.com,trisha@example.com,Trisha,Cruse,billing;secondaryBilling") + end + it 'returns voting contacts', vcr: false do get "/export/contacts?type=voting", nil, admin_headers