From f1a57be1a549dadb9e1b6c4c353c36c212e050a6 Mon Sep 17 00:00:00 2001 From: kjgarza Date: Thu, 4 Nov 2021 15:03:47 +0100 Subject: [PATCH 1/3] added created aggregation its difficult to see if events from crossref or event datacie have been created lated and where the gaps happen. the occured aggregation only focuses on when the events dates were generated but no when they were captured. this aggregation is more useful to identify problems with collection. --- app/controllers/events_controller.rb | 5 +++++ app/models/event.rb | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 5e4b445c3..f549b3484 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -185,6 +185,10 @@ def index if total > 0 facet_by_year(response.aggregations.occurred.buckets) end + created = + if total > 0 + facet_by_year(response.aggregations.created.buckets) + end prefixes = if total.positive? facet_by_key(response.response.aggregations.prefixes.buckets) @@ -219,6 +223,7 @@ def index page[:cursor].nil? && page[:number].present? ? page[:number] : nil, sources: sources, occurred: occurred, + createdAt: created, prefixes: prefixes, "citationTypes" => citation_types, "relationTypes" => relation_types, diff --git a/app/models/event.rb b/app/models/event.rb index 6732a640c..fe1c2af82 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -263,6 +263,16 @@ def self.query_aggregations }, aggs: { bucket_truncate: { bucket_sort: { size: 10 } } }, }, + created: { + date_histogram: { + field: "created_at", + interval: "year", + format: "year", + order: { _key: "desc" }, + min_doc_count: 1, + }, + aggs: { bucket_truncate: { bucket_sort: { size: 10 } } }, + }, registrants: { terms: { field: "registrant_id", size: 10, min_doc_count: 1 }, aggs: { From b9d19147ed951182fafa713acac237fa87261585 Mon Sep 17 00:00:00 2001 From: kjgarza Date: Thu, 11 Nov 2021 18:14:36 +0100 Subject: [PATCH 2/3] added test --- spec/requests/events_spec.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index 1802ca357..7189515f5 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -364,6 +364,12 @@ "datacite.crossref.citations", ) end + + it "has created aggregation" do + get uri, nil, headers + expect(json.dig("meta", "created", 0, "count")).to eq(1) + expect(json.dig("meta", "occurred", 0, "count")).to eq(1) + end end context "with nested attributes" do From 037d3ba6cb4a5241d482452337ca9996f37ee828 Mon Sep 17 00:00:00 2001 From: kjgarza Date: Tue, 21 Dec 2021 15:22:05 +0100 Subject: [PATCH 3/3] fix test --- app/controllers/events_controller.rb | 2 +- .../has_created_aggregation.yml | 101 ++++++++++++++++++ spec/requests/events_spec.rb | 5 + 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/vcr_cassettes/EventsController/create/with_registrant_information/has_created_aggregation.yml diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index f549b3484..b4a938b82 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -223,7 +223,7 @@ def index page[:cursor].nil? && page[:number].present? ? page[:number] : nil, sources: sources, occurred: occurred, - createdAt: created, + created: created, prefixes: prefixes, "citationTypes" => citation_types, "relationTypes" => relation_types, diff --git a/spec/fixtures/vcr_cassettes/EventsController/create/with_registrant_information/has_created_aggregation.yml b/spec/fixtures/vcr_cassettes/EventsController/create/with_registrant_information/has_created_aggregation.yml new file mode 100644 index 000000000..f567bf0e8 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/EventsController/create/with_registrant_information/has_created_aggregation.yml @@ -0,0 +1,101 @@ +--- +http_interactions: +- request: + method: get + uri: https://doi.org/ra/10.18713 + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; Maremma/4.9.4; mailto:info@datacite.org) + Accept: + - text/html,application/json,application/xml;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Accept-Encoding: + - gzip,deflate + response: + status: + code: 200 + message: '' + headers: + Date: + - Tue, 21 Dec 2021 14:21:42 GMT + Content-Type: + - application/json;charset=UTF-8 + Connection: + - keep-alive + Content-Encoding: + - gzip + Cf-Cache-Status: + - DYNAMIC + Expect-Ct: + - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct" + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=lFY4bnX9IEEUgQqkhEQ7BOG2%2FIlH5PaN0olNdRBhi07wOPKxvNBFIdcfRdNyNJfCNldUB3ecl9Lf6giP%2BN93BQDFAl77enLJZMHI9Vd6QCTrxKFaYurE2Xc%3D"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Server: + - cloudflare + Cf-Ray: + - 6c11c221cc668b93-FRA + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA4vmUlCoBmIFBSUXf08lKwUlQwM9QwtzQ2MlHYhwkCNI1CWxJNE5syRVCShYyxULADrjNVo3AAAA + http_version: null + recorded_at: Wed, 08 Apr 2015 00:00:00 GMT +- request: + method: get + uri: https://doi.org/ra/10.1016 + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; Maremma/4.9.4; mailto:info@datacite.org) + Accept: + - text/html,application/json,application/xml;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Accept-Encoding: + - gzip,deflate + response: + status: + code: 200 + message: '' + headers: + Date: + - Tue, 21 Dec 2021 14:21:42 GMT + Content-Type: + - application/json;charset=UTF-8 + Connection: + - keep-alive + Content-Encoding: + - gzip + Cf-Cache-Status: + - DYNAMIC + Expect-Ct: + - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct" + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=PDnklYEQGsQ5G5RM%2BKVslm2UN%2F7%2BnJf80Lxd%2FE4gALjlxaAPn4e22C8QdQ8o2%2BaaYrxJ8guvWrk9WllbBHgTJWODMmvY4eiJH28AoUJI5sMpIFaS08o2JQE%3D"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Server: + - cloudflare + Cf-Ray: + - 6c11c222cca9d6c5-FRA + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA4vmUlCoBmIFBSUXf08lKwUlQwM9QwNDMyUdiGiQI0jQuSi/uLgoNU0JKFjLFQsAH0F06DYAAAA= + http_version: null + recorded_at: Wed, 08 Apr 2015 00:00:00 GMT +recorded_with: VCR 5.1.0 diff --git a/spec/requests/events_spec.rb b/spec/requests/events_spec.rb index 7189515f5..d98f568d4 100644 --- a/spec/requests/events_spec.rb +++ b/spec/requests/events_spec.rb @@ -366,7 +366,12 @@ end it "has created aggregation" do + post uri, params, headers + + Event.import + sleep 2 get uri, nil, headers + expect(json.dig("meta", "created", 0, "count")).to eq(1) expect(json.dig("meta", "occurred", 0, "count")).to eq(1) end