From 827e099cdcd41512c4395d60fd7aeb0a10afd9d4 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 19 Mar 2019 15:08:06 +0100 Subject: [PATCH 1/3] separate aggreagtions for totals to review performance --- app/controllers/clients_controller.rb | 2 +- app/controllers/prefixes_controller.rb | 2 +- app/controllers/providers_controller.rb | 4 +++- app/models/concerns/indexable.rb | 2 +- app/models/doi.rb | 11 ++++++++--- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app/controllers/clients_controller.rb b/app/controllers/clients_controller.rb index 72dc8d148..8be527afb 100644 --- a/app/controllers/clients_controller.rb +++ b/app/controllers/clients_controller.rb @@ -131,7 +131,7 @@ def destroy def totals page = { size: 0, number: 1} - response = Doi.query(nil, provider_id: params[:provider_id], page: page) + response = Doi.query(nil, provider_id: params[:provider_id], page: page, totals_agg: true) total = response.results.total registrant = total > 0 ? clients_totals(response.response.aggregations.clients_totals.buckets) : nil diff --git a/app/controllers/prefixes_controller.rb b/app/controllers/prefixes_controller.rb index 57a662cc8..cc1a241c4 100644 --- a/app/controllers/prefixes_controller.rb +++ b/app/controllers/prefixes_controller.rb @@ -138,7 +138,7 @@ def update def totals page = { size: 0, number: 1} - response = Doi.query("", client_id: params[:client_id],page: page) + response = Doi.query("", client_id: params[:client_id],page: page, totals_agg: true) total = response.results.total registrant = total > 0 ? prefixes_totals(response.response.aggregations.prefixes_totals.buckets) : nil diff --git a/app/controllers/providers_controller.rb b/app/controllers/providers_controller.rb index c34cdea54..c8d5bfa95 100644 --- a/app/controllers/providers_controller.rb +++ b/app/controllers/providers_controller.rb @@ -122,7 +122,9 @@ def update def totals page = { size: 0, number: 1} - response = Doi.query("", page: page) + + response = Doi.query("", page: page, totals_agg: true) + total = response.results.total registrant = total > 0 ? providers_totals(response.response.aggregations.providers_totals.buckets) : nil diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index d404ecc23..29b2b97b4 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -100,7 +100,7 @@ def find_by_ids(ids, options={}) end def query(query, options={}) - aggregations = query_aggregations + aggregations = options[:totals_agg] == true ? totals_aggregations : query_aggregations options[:page] ||= { size: 25, number: 1 } # enable cursor-based pagination for DOIs diff --git a/app/models/doi.rb b/app/models/doi.rb index 5bea93851..8f5b35faa 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -326,10 +326,15 @@ def self.query_aggregations link_checks_citation_doi: { value_count: { field: "landing_page.citationDoi" } }, links_checked: { value_count: { field: "landing_page.checked" } }, sources: { terms: { field: 'source', size: 15, min_doc_count: 1 } }, - providers_totals: { terms: { field: 'provider_id', size: 500, min_doc_count: 1 }, aggs: sub_aggregations}, + subjects: { terms: { field: 'subjects.subject', size: 15, min_doc_count: 1 } } + } + end + + def self.totals_aggregations + { + providers_totals: { terms: { field: 'provider_id', size: 200, min_doc_count: 1 }, aggs: sub_aggregations}, clients_totals: { terms: { field: 'client_id', size: 2000, min_doc_count: 1 }, aggs: sub_aggregations }, prefixes_totals: { terms: { field: 'prefix', size: 2000, min_doc_count: 1 }, aggs: sub_aggregations }, - subjects: { terms: { field: 'subjects.subject', size: 15, min_doc_count: 1 } } } end @@ -339,7 +344,7 @@ def self.sub_aggregations beginning_of_month = DateTime.current.beginning_of_month.strftime('%Y-%m-%d') { - states: { terms: { field: 'aasm_state', size: 15, min_doc_count: 1 } }, + states: { terms: { field: 'aasm_state', size: 4, min_doc_count: 1 } }, this_month:{ date_range:{ field: 'created', time_zone: "CET", ranges: {from: beginning_of_month, to: "now/d"} } }, this_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: beginning_of_year, to: "now/d"} } }, last_year: { date_range: { field: 'created', time_zone: "CET", ranges: {from: beginning_of_last_year, to: beginning_of_year} } } From 2280eff1227b1c8d4710c98749b09ff9b6562342 Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Tue, 19 Mar 2019 16:08:15 +0100 Subject: [PATCH 2/3] Added some readonly options + added event attribute for writeOnly --- openapi.yaml | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index ca67fbd91..a013d80bf 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -828,8 +828,10 @@ components: type: string created: type: string + readOnly: true updated: type: string + readOnly: true required: - id - attributes - name - symbol - contactName - contactEmail @@ -852,11 +854,14 @@ components: properties: created: type: string + readOnly: true updated: type: string + readOnly: true required: - id Doi: + description: Represents a DOI and provides access to metadata attributes, further schema specific information can be found at https://schema.datacite.org type: object properties: data: @@ -875,6 +880,11 @@ components: type: string suffix: type: string + event: + description: Can be set to trigger a DOI state change. + type: string + enum: [publish, register, hide] + writeOnly: true identifiers: type: array items: @@ -1101,10 +1111,12 @@ components: isActive: type: boolean state: + readOnly: true type: string reason: type: string landingPage: + description: Data describing the landing page, used by link checking. type: object properties: checked: @@ -1135,10 +1147,13 @@ components: type: boolean created: type: string + readOnly: true registered: type: string + readOnly: true updated: type: string + readOnly: true required: - id - attributes - doi - creators - titles - publisher - publicationYear @@ -1226,8 +1241,10 @@ components: properties: created: type: string + readOnly: true updated: type: string + readOnly: true required: - id Provider: @@ -1269,8 +1286,8 @@ components: report-id: type: string description: >- - The report ID or code or shortname. Typically this will be the same - code provided in the Report parameter of the request. + The report ID or code or shortname. Typically this will be the + same code provided in the Report parameter of the request. example: DSR release: type: string @@ -1290,14 +1307,14 @@ components: report-filters: type: string description: >- - Zero or more report filters used for this report. Typically reflect - filters provided on the Request. Filters limit the data to be - reported on. + Zero or more report filters used for this report. Typically + reflect filters provided on the Request. Filters limit the data + to be reported on. report-attributes: type: string description: >- - Zero or more additional attributes applied to the report. Attributes - inform the level of detail in the report. + Zero or more additional attributes applied to the report. + Attributes inform the level of detail in the report. reporting-period: type: string description: Time the report was prepared. From 76407c0028286e7448ae3e94f1356b644751ba49 Mon Sep 17 00:00:00 2001 From: Kristian Garza Date: Tue, 19 Mar 2019 16:36:33 +0100 Subject: [PATCH 3/3] limit the size of agg --- app/models/concerns/indexable.rb | 4 ++++ app/models/doi.rb | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/indexable.rb b/app/models/concerns/indexable.rb index 29b2b97b4..eadb1b2b3 100644 --- a/app/models/concerns/indexable.rb +++ b/app/models/concerns/indexable.rb @@ -243,6 +243,10 @@ def recreate_index(options={}) client.indices.create index: index_name, body: { settings: {"index.requests.cache.enable": true }} end + def count + Elasticsearch::Model.client.count(index: index_name)['count'] + end + def create_alias(index: nil) return nil unless index.present? diff --git a/app/models/doi.rb b/app/models/doi.rb index 8f5b35faa..b765cd512 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -332,9 +332,9 @@ def self.query_aggregations def self.totals_aggregations { - providers_totals: { terms: { field: 'provider_id', size: 200, min_doc_count: 1 }, aggs: sub_aggregations}, - clients_totals: { terms: { field: 'client_id', size: 2000, min_doc_count: 1 }, aggs: sub_aggregations }, - prefixes_totals: { terms: { field: 'prefix', size: 2000, min_doc_count: 1 }, aggs: sub_aggregations }, + providers_totals: { terms: { field: 'provider_id', size: ::Provider.count, min_doc_count: 1 }, aggs: sub_aggregations}, + clients_totals: { terms: { field: 'client_id', size: ::Client.count, min_doc_count: 1 }, aggs: sub_aggregations }, + prefixes_totals: { terms: { field: 'prefix', size: ::Prefix.count, min_doc_count: 1 }, aggs: sub_aggregations }, } end