Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactors /dois faceting and adds support for on-demand facets #1299

Merged
merged 4 commits into from
Dec 12, 2024
Merged

Refactors /dois faceting and adds support for on-demand facets #1299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7283a46
Refactors /dois faceting and adds support for on-demand facets
codycooperross Dec 6, 2024
5b3fc65
Support for facets and disable-facets URL params in find_by_ids
codycooperross Dec 6, 2024
642b221
Change enumeration to hash method
codycooperross Dec 9, 2024
f4029e1
Implements expected behavior for disable_facets=false and allow query…
codycooperross Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 41 additions & 62 deletions app/controllers/datacite_dois_controller.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def index
if params[:id].present?
response = DataciteDoi.find_by_id(params[:id])
elsif params[:ids].present?
response = DataciteDoi.find_by_ids(params[:ids], page: page, sort: sort)
response = DataciteDoi.find_by_ids(params[:ids], disable_facets: params[:disable_facets], facets: params[:facets], page: page, sort: sort)
else
response =
DataciteDoi.query(
Expand Down Expand Up @@ -136,6 +136,7 @@ def index
source: params[:source],
scroll_id: params[:scroll_id],
disable_facets: disable_facets,
facets: params[:facets],
page: page,
sort: sort,
random: params[:random],
Expand Down Expand Up @@ -221,47 +222,43 @@ def index
)
end
else
if total.positive? && !disable_facets
states = facet_by_key(response.aggregations.states.buckets)
resource_types = facet_by_combined_key(response.aggregations.resource_types.buckets)
published = facet_by_range(response.aggregations.published.buckets)
created = facet_by_key_as_string(response.aggregations.created.buckets)
created_by_month = response.aggregations.created_by_month ? facet_by_key_as_string(response.aggregations.created_by_month.buckets) : nil
registered = facet_by_key_as_string(response.aggregations.registered.buckets)
providers = facet_by_combined_key(response.aggregations.providers.buckets)
clients = facet_by_combined_key(response.aggregations.clients.buckets)
prefixes = facet_by_key(response.aggregations.prefixes.buckets)
schema_versions = facet_by_schema(response.aggregations.schema_versions.buckets)
affiliations = facet_by_combined_key(response.aggregations.affiliations.buckets)
subjects = facet_by_key(response.aggregations.subjects.buckets)
fields_of_science = facet_by_fos(response.aggregations.fields_of_science.subject.buckets)
certificates = facet_by_key(response.aggregations.certificates.buckets)
licenses = facet_by_license(response.aggregations.licenses.buckets)
link_checks_status = facet_by_cumulative_year(response.aggregations.link_checks_status.buckets)
citations = metric_facet_by_year(response.aggregations.citations.buckets)
views = metric_facet_by_year(response.aggregations.views.buckets)
downloads = metric_facet_by_year(response.aggregations.downloads.buckets)
else
states = nil
resource_types = nil
published = nil
created = nil
registered = nil
providers = nil
clients = nil
prefixes = nil
schema_versions = nil
affiliations = nil
subjects = nil
fields_of_science = nil
certificates = nil
licenses = nil
link_checks_status = nil
citations = nil
views = nil
downloads = nil
end
facets_to_facet_methods = {
states: :facet_by_key,
resource_types: :facet_by_combined_key,
created: :facet_by_key_as_string,
created_by_month: :facet_by_key_as_string,
published: :facet_by_range,
registered: :facet_by_key_as_string,
providers: :facet_by_combined_key,
clients: :facet_by_combined_key,
client_types: :facet_by_client_type,
affiliations: :facet_by_combined_key,
prefixes: :facet_by_key,
certificates: :facet_by_key,
licenses: :facet_by_license,
schema_versions: :facet_by_schema,
link_checks_status: :facet_by_cumulative_year,
creators_and_contributors: :facet_by_creators_and_contributors,
subjects: :facet_by_key,
fields_of_science: :facet_by_fos,
languages: :facet_by_language,
registration_agencies: :facet_by_registration_agency,
citations: :metric_facet_by_year,
views: :metric_facet_by_year,
downloads: :metric_facet_by_year
}

facets_to_bucket_path = {
fields_of_science: [:subject, :buckets]
}

aggregations = response.aggregations
facets = facets_to_facet_methods.map do |facet, method|
if aggregations.dig(facet)
buckets = facets_to_bucket_path.dig(facet) ? aggregations.dig(facet, *facets_to_bucket_path[facet]) : aggregations.dig(facet).buckets
[facet.to_s.camelize(:lower), send(method, buckets)]
end
end.compact.to_h
respond_to do |format|
format.json do
options = {}
Expand All @@ -271,27 +268,8 @@ def index
page:
if page[:cursor].nil? && page[:number].present?
page[:number]
end,
states: states,
"resourceTypes" => resource_types,
created: created,
createdByMonth: created_by_month,
published: published,
registered: registered,
providers: providers,
clients: clients,
affiliations: affiliations,
prefixes: prefixes,
certificates: certificates,
licenses: licenses,
"schemaVersions" => schema_versions,
"linkChecksStatus" => link_checks_status,
subjects: subjects,
"fieldsOfScience" => fields_of_science,
citations: citations,
views: views,
downloads: downloads,
}.compact
end
}.merge(facets).compact

options[:links] = {
self: request.original_url,
Expand Down Expand Up @@ -325,6 +303,7 @@ def index
"has-affiliation" => params[:has_affiliation],
"has-funder" => params[:has_funder],
"disable-facets" => params[:disable_facets],
"facets" => params[:facets],
detail: params[:detail],
composite: params[:composite],
affiliation: params[:affiliation],
Expand Down
202 changes: 131 additions & 71 deletions app/models/doi.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -674,82 +674,142 @@ def as_indexed_json(_options = {})
}
end


def self.query_aggregations(disable_facets: false)
if !disable_facets
{
# number of resourceTypeGeneral increased from 16 to 28 in schema 4.4
resource_types: { terms: { field: "resource_type_id_and_name", size: 30, min_doc_count: 1 } },
states: { terms: { field: "aasm_state", size: 3, min_doc_count: 1 } },
published: {
date_histogram: {
field: "publication_year",
interval: "year",
format: "year",
order: {
_key: "desc",
},
min_doc_count: 1,
DOI_AGGREGATION_DEFINITIONS = {
# number of resourceTypeGeneral increased from 30 to 32 in schema 4.6
resource_types: { terms: { field: "resource_type_id_and_name", size: 32, min_doc_count: 1 } },
states: { terms: { field: "aasm_state", size: 3, min_doc_count: 1 } },
published: {
date_histogram: {
field: "publication_year",
interval: "year",
format: "year",
order: {
_key: "desc",
},
min_doc_count: 1,
},
registration_agencies: { terms: { field: "agency", size: 10, min_doc_count: 1 } },
created: { date_histogram: { field: "created", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: { bucket_truncate: { bucket_sort: { size: 10 } } } },
registered: { date_histogram: { field: "registered", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: { bucket_truncate: { bucket_sort: { size: 10 } } } },
providers: { terms: { field: "provider_id_and_name", size: 10, min_doc_count: 1 } },
clients: { terms: { field: "client_id_and_name", size: 10, min_doc_count: 1 } },
affiliations: { terms: { field: "affiliation_id_and_name", size: 10, min_doc_count: 1 } },
prefixes: { terms: { field: "prefix", size: 10, min_doc_count: 1 } },
schema_versions: { terms: { field: "schema_version", size: 10, min_doc_count: 1 } },
link_checks_status: { terms: { field: "landing_page.status", size: 10, min_doc_count: 1 } },
# link_checks_has_schema_org: { terms: { field: 'landing_page.hasSchemaOrg', size: 2, min_doc_count: 1 } },
# link_checks_schema_org_id: { value_count: { field: "landing_page.schemaOrgId" } },
# link_checks_dc_identifier: { value_count: { field: "landing_page.dcIdentifier" } },
# link_checks_citation_doi: { value_count: { field: "landing_page.citationDoi" } },
# links_checked: { value_count: { field: "landing_page.checked" } },
# sources: { terms: { field: 'source', size: 15, min_doc_count: 1 } },
subjects: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1 } },
pid_entities: {
filter: { term: { "subjects.subjectScheme": "PidEntity" } },
aggs: {
subject: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1,
include: %w(Dataset Publication Software Organization Funder Person Grant Sample Instrument Repository Project) } },
},
},
registration_agencies: { terms: { field: "agency", size: 10, min_doc_count: 1 } },
created: { date_histogram: { field: "created", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: { bucket_truncate: { bucket_sort: { size: 10 } } } },
registered: { date_histogram: { field: "registered", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: { bucket_truncate: { bucket_sort: { size: 10 } } } },
providers: { terms: { field: "provider_id_and_name", size: 10, min_doc_count: 1 } },
clients: { terms: { field: "client_id_and_name", size: 10, min_doc_count: 1 } },
client_types: {
terms: {
field: "client.client_type",
size: 4,
min_doc_count: 1
}
},
affiliations: { terms: { field: "affiliation_id_and_name", size: 10, min_doc_count: 1 } },
prefixes: { terms: { field: "prefix", size: 10, min_doc_count: 1 } },
schema_versions: { terms: { field: "schema_version", size: 10, min_doc_count: 1 } },
link_checks_status: { terms: { field: "landing_page.status", size: 10, min_doc_count: 1 } },
subjects: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1 } },
pid_entities: {
filter: { term: { "subjects.subjectScheme": "PidEntity" } },
aggs: {
subject: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1,
include: %w(Dataset Publication Software Organization Funder Person Grant Sample Instrument Repository Project) } },
},
fields_of_science: {
filter: { term: { "subjects.subjectScheme": "Fields of Science and Technology (FOS)" } },
aggs: {
subject: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1,
include: "FOS:.*" } },
},
},
fields_of_science: {
filter: { term: { "subjects.subjectScheme": "Fields of Science and Technology (FOS)" } },
aggs: {
subject: { terms: { field: "subjects.subject", size: 10, min_doc_count: 1,
include: "FOS:.*" } },
},
licenses: { terms: { field: "rights_list.rightsIdentifier", size: 10, min_doc_count: 1 } },
languages: { terms: { field: "language", size: 10, min_doc_count: 1 } },
certificates: { terms: { field: "client.certificate", size: 10, min_doc_count: 1 } },
views: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "view_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
},
},
licenses: { terms: { field: "rights_list.rightsIdentifier", size: 10, min_doc_count: 1 } },
languages: { terms: { field: "language", size: 10, min_doc_count: 1 } },
certificates: { terms: { field: "client.certificate", size: 10, min_doc_count: 1 } },
creators_and_contributors: {
terms: {
field: "creators_and_contributors.nameIdentifiers.nameIdentifier",
size: 10,
min_doc_count: 1,
include: "https?://orcid.org/.*"
},
downloads: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "download_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
aggs: {
creators_and_contributors: {
top_hits: {
_source: {
includes: [
"creators_and_contributors.name",
"creators_and_contributors.nameIdentifiers.nameIdentifier"
]
},
size: 1
}
},
"work_types": {
"terms": {
"field": "resource_type_id_and_name",
"min_doc_count": 1
}
}
}
},
views: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "view_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
},
citations: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "citation_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
},
},
downloads: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "download_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
},
}
end
},
citations: {
date_histogram: { field: "publication_year", interval: "year", format: "year", order: { _key: "desc" }, min_doc_count: 1 },
aggs: {
metric_count: { sum: { field: "citation_count" } },
bucket_truncate: { bucket_sort: { size: 10 } },
},
},
}

def self.default_doi_query_facets
[
:resource_types,
:states,
:published,
:created,
:registered,
:providers,
:clients,
:affiliations,
:prefixes,
:schema_versions,
:link_checks_status,
:subjects,
:fields_of_science,
:licenses,
:certificates,
:views,
:downloads,
:citations
]
end

def self.query_aggregations(disable_facets: false, facets: nil)
return {} if disable_facets.to_s == "true"

selected_facets = if facets.is_a?(String)
facets.split(",").map(&:strip).map(&:underscore).map(&:to_sym)
else
Array.wrap(facets).map(&:to_sym)
end.uniq

selected_facets = default_doi_query_facets if facets.nil?

DOI_AGGREGATION_DEFINITIONS.slice(*selected_facets)
end

def self.provider_aggregations
Expand Down Expand Up @@ -820,7 +880,7 @@ def self.find_by_ids(ids, options = {})
must: must,
},
},
aggregations: query_aggregations(disable_facets: options[:disable_facets]),
aggregations: query_aggregations(disable_facets: options[:disable_facets], facets: options[:facets]),
)
end

Expand Down Expand Up @@ -902,9 +962,9 @@ def self.query(query, options = {})
elsif options[:totals_agg] == "prefix"
prefix_aggregations
elsif options[:client_type] == "igsnCatalog"
query_aggregations(disable_facets: options[:disable_facets]).merge(self.igsn_id_catalog_aggregations)
query_aggregations(disable_facets: options[:disable_facets], facets: options[:facets]).merge(self.igsn_id_catalog_aggregations)
else
query_aggregations(disable_facets: options[:disable_facets])
query_aggregations(disable_facets: options[:disable_facets], facets: options[:facets])
end

# Cursor nav uses search_after, this should always be an array of values that match the sort.
Expand Down
Loading
Loading