diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index 48a6bce8c..29fb47266 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -186,16 +186,6 @@ type Audiovisual implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -505,16 +495,6 @@ type Book implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -779,16 +759,6 @@ type BookChapter implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -1183,16 +1153,6 @@ type Collection implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -1501,16 +1461,6 @@ type ConferencePaper implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -2112,16 +2062,6 @@ type DataManagementPlan implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -2431,16 +2371,6 @@ type DataPaper implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -2749,16 +2679,6 @@ type Dataset implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -3186,16 +3106,6 @@ type Dissertation implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -3508,16 +3418,6 @@ interface DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -3832,16 +3732,6 @@ type Event implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -4135,16 +4025,6 @@ type EventData implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -4734,16 +4614,6 @@ type Image implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -5051,16 +4921,6 @@ type Instrument implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -5368,16 +5228,6 @@ type InteractiveResource implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -5691,16 +5541,6 @@ type JournalArticle implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -6305,16 +6145,6 @@ type Model implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -6795,16 +6625,6 @@ type Other implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -7138,16 +6958,6 @@ type PeerReview implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -7593,16 +7403,6 @@ type PhysicalObject implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -7964,16 +7764,6 @@ type Preprint implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -8282,16 +8072,6 @@ type Publication implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -9245,16 +9025,6 @@ type Service implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -9564,16 +9334,6 @@ type Software implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -9913,16 +9673,6 @@ type Sound implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -10442,16 +10192,6 @@ type Work implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ @@ -10651,8 +10391,6 @@ type WorkConnectionWithTotal { """ edges: [WorkEdge] fieldsOfScience: [Facet!] - fieldsOfScienceCombined: [Facet!] - fieldsOfScienceRepository: [Facet!] languages: [Facet!] licenses: [Facet!] @@ -10768,16 +10506,6 @@ type Workflow implements DoiItem { """ fieldsOfScience: [FieldOfScience!] - """ - OECD Fields of Science of the resource and containing repository - """ - fieldsOfScienceCombined: [FieldOfScience!] - - """ - OECD Fields of Science of the containing repository - """ - fieldsOfScienceRepository: [FieldOfScience!] - """ Technical format of the resource """ diff --git a/app/graphql/types/doi_item.rb b/app/graphql/types/doi_item.rb index a96c36d11..b4134356e 100644 --- a/app/graphql/types/doi_item.rb +++ b/app/graphql/types/doi_item.rb @@ -66,15 +66,6 @@ module DoiItem field :fields_of_science, [FieldOfScienceType], null: true, description: "OECD Fields of Science of the resource" - - field :fields_of_science_combined, - [FieldOfScienceType], - null: true, description: "OECD Fields of Science of the resource and containing repository" - - field :fields_of_science_repository, - [FieldOfScienceType], - null: true, description: "OECD Fields of Science of the containing repository" - field :dates, [DateType], null: true, description: "Different dates relevant to the work" @@ -427,33 +418,14 @@ def registration_agency { id: object.agency, name: REGISTRATION_AGENCIES[object.agency] }.compact end - def _fos_to_facet(fos_list) - Array.wrap(fos_list).map do |name| - { "id" => name.parameterize(separator: "_"), "name" => name } - end.uniq - end - - def fields_of_science_repository - if object.client.blank? - return [] - end - _fos_to_facet(object.fields_of_science_repository) - end - - def fields_of_science_combined - _fos_to_facet(object.fields_of_science_combined) - end - - def _fos_temp + def fields_of_science Array.wrap(object.subjects).select do |s| s["subjectScheme"] == "Fields of Science and Technology (FOS)" end.map do |s| - s["subject"].gsub("FOS: ", "") + name = s["subject"].gsub("FOS: ", "") + { "id" => name.parameterize(separator: "_"), "name" => name } end.uniq end - def fields_of_science - _fos_to_facet(_fos_temp) - end def creators(**args) Array.wrap(object.creators)[0...args[:first]].map do |c| diff --git a/app/graphql/types/query_type.rb b/app/graphql/types/query_type.rb index 3fe3124b7..22f2d2d76 100644 --- a/app/graphql/types/query_type.rb +++ b/app/graphql/types/query_type.rb @@ -277,12 +277,7 @@ def actor(id:) end def works(**args) - ElasticsearchModelResponseConnection.new( - response(args), { - context: context, - first: args[:first], - after: args[:after] - }) + ElasticsearchModelResponseConnection.new(response(args), context: context, first: args[:first], after: args[:after]) end field :work, WorkType, null: false do @@ -1258,37 +1253,7 @@ def usage_report(id:) end def response(**args) - Doi.gql_query( - args[:query], - ids: args[:ids], - user_id: args[:user_id], - client_id: args[:repository_id], - provider_id: args[:member_id], - resource_type_id: args[:resource_type_id], - resource_type: args[:resource_type], - published: args[:published], - agency: args[:registration_agency], - language: args[:language], - license: args[:license], - has_person: args[:has_person], - has_funder: args[:has_funder], - has_organization: args[:has_organization], - has_affiliation: args[:has_affiliation], - has_member: args[:has_member], - has_citations: args[:has_citations], - has_parts: args[:has_parts], - has_versions: args[:has_versions], - has_views: args[:has_views], - has_downloads: args[:has_downloads], - field_of_science: args[:field_of_science], - facet_count: args[:facet_count], - pid_entity: args[:pid_entity], - state: "findable", - page: { - cursor: args[:after].present? ? Base64.urlsafe_decode64(args[:after]) : [], - size: args[:first] - } - ) + Doi.gql_query(args[:query], ids: args[:ids], user_id: args[:user_id], client_id: args[:repository_id], provider_id: args[:member_id], resource_type_id: args[:resource_type_id], resource_type: args[:resource_type], published: args[:published], agency: args[:registration_agency], language: args[:language], license: args[:license], has_person: args[:has_person], has_funder: args[:has_funder], has_organization: args[:has_organization], has_affiliation: args[:has_affiliation], has_member: args[:has_member], has_citations: args[:has_citations], has_parts: args[:has_parts], has_versions: args[:has_versions], has_views: args[:has_views], has_downloads: args[:has_downloads], field_of_science: args[:field_of_science], facet_count: args[:facet_count], pid_entity: args[:pid_entity], state: "findable", page: { cursor: args[:after].present? ? Base64.urlsafe_decode64(args[:after]) : [], size: args[:first] }) end def set_doi(id) diff --git a/app/graphql/types/work_connection_with_total_type.rb b/app/graphql/types/work_connection_with_total_type.rb index 7fb8d4436..b0c5b58d4 100644 --- a/app/graphql/types/work_connection_with_total_type.rb +++ b/app/graphql/types/work_connection_with_total_type.rb @@ -17,8 +17,6 @@ class WorkConnectionWithTotalType < BaseConnection field :affiliations, [FacetType], null: true, cache: true field :authors, [FacetType], null: true, cache: true field :fields_of_science, [FacetType], null: true, cache: true - field :fields_of_science_combined, [FacetType], null: true, cache: true - field :fields_of_science_repository, [FacetType], null: true, cache: true field :licenses, [FacetType], null: true, cache: true field :languages, [FacetType], null: true, cache: true @@ -108,22 +106,6 @@ def fields_of_science end end - def fields_of_science_combined - if object.aggregations.fields_of_science_combined - facet_by_fos(object.aggregations.fields_of_science_combined.buckets) - else - [] - end - end - - def fields_of_science_repository - if object.aggregations.fields_of_science_repository - facet_by_fos(object.aggregations.fields_of_science_repository.buckets) - else - [] - end - end - def languages if object.aggregations.languages facet_by_language(object.aggregations.languages.buckets) diff --git a/app/models/client.rb b/app/models/client.rb index 6b1dcdfc6..5e1452e8a 100644 --- a/app/models/client.rb +++ b/app/models/client.rb @@ -52,7 +52,6 @@ class Client < ApplicationRecord attr_accessor :password_input, :target_id attr_reader :from_salesforce - validate :subjects_only_for_disciplinary_repos validates :subjects, if: :subjects?, json: { message: ->(errors) { errors }, @@ -97,16 +96,12 @@ class Client < ApplicationRecord has_many :activities, as: :auditable, dependent: :destroy before_validation :set_defaults - before_validation :convert_subject_hashes_to_camelcase before_create { self.created = Time.zone.now.utc.iso8601 } before_save { self.updated = Time.zone.now.utc.iso8601 } after_create :assign_prefix after_create_commit :create_reference_repository after_update_commit :update_reference_repository after_destroy_commit :destroy_reference_repository - after_commit on: %i[update] do - ::Client.import_dois(self.symbol) - end # use different index for testing if Rails.env.test? @@ -401,7 +396,6 @@ def as_indexed_json(options = {}) end, "analytics_dashboard_url" => analytics_dashboard_url, "analytics_tracking_id" => analytics_tracking_id, - "subjects" => Array.wrap(subjects), } end @@ -510,7 +504,7 @@ def re3data=(value) end def subjects=(value) - write_attribute(:subjects, Array.wrap(value).uniq) + write_attribute(:subjects, Array.wrap(value)) end def opendoar=(value) @@ -916,15 +910,6 @@ def freeze_symbol errors.add(:symbol, "cannot be changed") if symbol_changed? end - def subjects_only_for_disciplinary_repos - if Array.wrap(subjects).any? && Array.wrap(repository_type).exclude?("disciplinary") - errors.add( - :subjects, - "Subjects are only allowed for disciplinary repositories. This repository_type is: #{repository_type}", - ) - end - end - def check_id if symbol && symbol.split(".").first != provider.symbol errors.add( @@ -970,7 +955,7 @@ def assign_prefix ClientPrefix.create( client_id: symbol, provider_prefix_id: provider_prefix.uid, - prefix_id: provider_prefix.prefix.uid + prefix_id: provider_prefix.prefix.uid, ) end end @@ -995,18 +980,6 @@ def set_defaults self.doi_quota_allowed = -1 unless doi_quota_allowed.to_i > 0 end - def convert_subject_hashes_to_camelcase - if self.subjects? - self.subjects = Array.wrap(self.subjects).map { |subject| - subject.transform_keys! do |key| - key.to_s.camelcase(:lower) - end - } - else - [] - end - end - def create_reference_repository ReferenceRepository.create_from_client(self) end diff --git a/app/models/datacite_doi.rb b/app/models/datacite_doi.rb index 132f37c3b..8c018925b 100644 --- a/app/models/datacite_doi.rb +++ b/app/models/datacite_doi.rb @@ -39,13 +39,12 @@ def self.import_by_ids(options = {}) DataciteDoi.where(type: "DataciteDoi").maximum(:id) ). to_i - batch_size = options[:batch_size] || 50 count = 0 # TODO remove query for type once STI is enabled # SQS message size limit is 256 kB, up to 2 GB with S3 DataciteDoi.where(type: "DataciteDoi").where(id: from_id..until_id). - find_in_batches(batch_size: batch_size) do |dois| + find_in_batches(batch_size: 50) do |dois| ids = dois.pluck(:id) DataciteDoiImportInBulkJob.perform_later(ids, index: index) count += ids.length diff --git a/app/models/doi.rb b/app/models/doi.rb index b0ccd9357..60c136feb 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -141,8 +141,6 @@ class Doi < ApplicationRecord before_save :set_defaults, :save_metadata before_create { self.created = Time.zone.now.utc.iso8601 } - FIELD_OF_SCIENCE_SCHEME = "Fields of Science and Technology (FOS)" - scope :q, ->(query) { where("dataset.doi = ?", query) } # use different index for testing @@ -410,14 +408,6 @@ class Doi < ApplicationRecord updated: { type: :date }, deleted_at: { type: :date }, cumulative_years: { type: :integer, index: "false" }, - subjects: { type: :object, properties: { - subjectScheme: { type: :keyword }, - subject: { type: :keyword }, - schemeUri: { type: :keyword }, - valueUri: { type: :keyword }, - lang: { type: :keyword }, - classificationCode: { type: :keyword }, - } } } indexes :provider, type: :object, properties: { id: { type: :keyword }, @@ -522,9 +512,6 @@ class Doi < ApplicationRecord titleType: { type: :keyword }, lang: { type: :keyword }, } - indexes :fields_of_science, type: :keyword - indexes :fields_of_science_combined, type: :keyword - indexes :fields_of_science_repository, type: :keyword end end @@ -580,9 +567,6 @@ def as_indexed_json(_options = {}) "sizes" => Array.wrap(sizes), "language" => language, "subjects" => Array.wrap(subjects), - "fields_of_science" => fields_of_science, - "fields_of_science_repository" => fields_of_science_repository, - "fields_of_science_combined" => fields_of_science_combined, "xml" => xml, "is_active" => is_active, "landing_page" => landing_page, @@ -1728,26 +1712,6 @@ def client_id client.symbol.downcase if client.present? end - def _fos_filter(subject_array) - Array.wrap(subject_array).select { |sub| - sub.dig("subjectScheme") == FIELD_OF_SCIENCE_SCHEME - }.map do |fos| - fos["subject"].gsub("FOS: ", "") - end - end - - def fields_of_science - _fos_filter(subjects).uniq - end - - def fields_of_science_repository - _fos_filter(client&.subjects).uniq - end - - def fields_of_science_combined - fields_of_science | fields_of_science_repository - end - def client_id_and_name "#{client_id}:#{client.name}" if client.present? end diff --git a/app/models/schemas/client/subjects.json b/app/models/schemas/client/subjects.json index 8ece1f194..7179c2243 100644 --- a/app/models/schemas/client/subjects.json +++ b/app/models/schemas/client/subjects.json @@ -5,18 +5,34 @@ "items": { "type": "object", "properties": { + "classification_code": { "type": "string" }, "classificationCode": { "type": "string" }, + "scheme_uri": { "type": "string" }, "schemeUri": { "type": "string" }, + "value_uri": { "type": "string" }, "valueUri": { "type": "string" }, "lang": { "type": "string" }, "subject": { "type": "string" }, + "subject_scheme": { "type": "string" }, "subjectScheme": { "type": "string" } }, - "required": [ - "classificationCode", - "schemeUri", - "subject", - "subjectScheme" + "oneOf": [ + { + "required": [ + "classification_code", + "scheme_uri", + "subject", + "subject_scheme" + ] + }, + { + "required": [ + "classificationCode", + "schemeUri", + "subject", + "subjectScheme" + ] + } ], "additionalProperties": false } diff --git a/db/seeds/development/base.seeds.rb b/db/seeds/development/base.seeds.rb index f9725fcfe..4a32778aa 100644 --- a/db/seeds/development/base.seeds.rb +++ b/db/seeds/development/base.seeds.rb @@ -24,19 +24,11 @@ client = Client.where(symbol: "DATACITE.TEST").first || FactoryBot.create( - :client_with_fos, + :client, provider: provider, symbol: ENV["MDS_USERNAME"], password_input: ENV["MDS_PASSWORD"], ) -if Prefix.where(uid: "10.14454").blank? - prefix = FactoryBot.create(:prefix, uid: "10.14454") - ### This creates both the client_prefix and the provider association - FactoryBot.create( - :client_prefix, - client_id: client.symbol, prefix_id: prefix.uid, - ) -end dois = FactoryBot.create_list(:doi, 10, client: client, state: "findable") FactoryBot.create_list(:event_for_datacite_related, 3, obj_id: dois.first.doi) FactoryBot.create_list(:event_for_datacite_usage, 2, obj_id: dois.first.doi) diff --git a/lib/tasks/datacite_doi.rake b/lib/tasks/datacite_doi.rake index 40fbf98c7..4191231f2 100644 --- a/lib/tasks/datacite_doi.rake +++ b/lib/tasks/datacite_doi.rake @@ -65,14 +65,8 @@ namespace :datacite_doi do task import: :environment do from_id = (ENV["FROM_ID"] || DataciteDoi.minimum(:id)).to_i until_id = (ENV["UNTIL_ID"] || DataciteDoi.maximum(:id)).to_i - batch_size = ENV["BATCH_SIZE"].nil? ? 50 : ENV["BATCH_SIZE"].to_i - - DataciteDoi.import_by_ids( - from_id: from_id, - until_id: until_id, - batch_size: batch_size, - index: ENV["INDEX"] || DataciteDoi.inactive_index - ) + + DataciteDoi.import_by_ids(from_id: from_id, until_id: until_id, index: ENV["INDEX"] || DataciteDoi.inactive_index) end desc "Import one datacite DOI" diff --git a/lib/tasks/enrich.rake b/lib/tasks/enrich.rake deleted file mode 100644 index 3bd087000..000000000 --- a/lib/tasks/enrich.rake +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -namespace :enrich do - desc "Enrich Clients with Subjects from re3data and converted to Field Of Science subjectScheme" - task client_subjects: :environment do - def all_clients_from_query(query: nil) - # Loop through all clients - page = { size: 1_000, number: 1 } - response = Client.query(query, page: page) - clients = response.records.to_a - - total = response.records.total - total_pages = page[:size] > 0 ? (total.to_f / page[:size]).ceil : 0 - - # keep going for all pages - page_num = 2 - while page_num <= total_pages - page = { size: 1_000, number: page_num } - response = self.query(query, page: page) - clients = clients + response.records.to_a - page_num += 1 - end - clients - end - - def enrich_client(client) - re3data = DataCatalog.find_by_id(client.re3data_id).fetch(:data, []).first - if re3data - subs = re3data.subjects - dfg_ids = subs.select { |subject| - subject.scheme == "DFG" - }.map { |subject| - subject.text.split.first - } - fos_subjects = Bolognese::Utils.dfg_ids_to_fos(dfg_ids) - client.subjects = fos_subjects.uniq - client.save - end - end - - puts "Searching for disciplinary repositories with re3data_ids without subjects" - clients = all_clients_from_query(query: "re3data_id:* AND -subjects:* AND -deleted_at:* AND repository_type:disciplinary") - puts "Found #{clients.count} repostitories." - if clients.count > 0 - puts "Enriching now..." - clients.each do |c| - enrich_client(c) - end - puts "Enrichment complete" - else - puts "Skipping enrichment" - end - end -end diff --git a/spec/factories/client.rb b/spec/factories/client.rb index c5a40efe6..16b82fe80 100644 --- a/spec/factories/client.rb +++ b/spec/factories/client.rb @@ -19,21 +19,6 @@ password_input { "12345" } is_active { true } - factory :client_with_fos do - repository_type { "disciplinary" } - subjects do - [ - { - subject: "Physical sciences", - valueUri: "", - schemeUri: "http://www.oecd.org/science/inno/38235147.pdf", - subjectScheme: "Fields of Science and Technology (FOS)", - classificationCode: "1001", - }, - ] - end - end - initialize_with { Client.where(symbol: symbol).first_or_initialize } end end diff --git a/spec/graphql/types/doi_item_spec.rb b/spec/graphql/types/doi_item_spec.rb index e88ef3ff3..2db00a772 100644 --- a/spec/graphql/types/doi_item_spec.rb +++ b/spec/graphql/types/doi_item_spec.rb @@ -13,9 +13,6 @@ it { is_expected.to have_field(:publicationYear).of_type("Int") } it { is_expected.to have_field(:publisher).of_type("String") } it { is_expected.to have_field(:subjects).of_type("[Subject!]") } - it { is_expected.to have_field(:fieldsOfScience).of_type("[FieldOfScience!]") } - it { is_expected.to have_field(:fieldsOfScienceRepository).of_type("[FieldOfScience!]") } - it { is_expected.to have_field(:fieldsOfScienceCombined).of_type("[FieldOfScience!]") } it { is_expected.to have_field(:dates).of_type("[Date!]") } it { is_expected.to have_field(:registered).of_type("ISO8601DateTime") } it { is_expected.to have_field(:language).of_type("Language") } diff --git a/spec/graphql/types/work_type_spec.rb b/spec/graphql/types/work_type_spec.rb index 6f5473628..30e5b63f2 100644 --- a/spec/graphql/types/work_type_spec.rb +++ b/spec/graphql/types/work_type_spec.rb @@ -1201,87 +1201,4 @@ ) end end - - describe "query works with repository subjects" do - before :all do - SLEEP_TIME = 2 - WORK_COUNT = 10 - - DataciteDoi.import(force: true) - Client.import(force: true) - Prefix.import(force: true) - ClientPrefix.import(force: true) - ReferenceRepository.import(force: true) - Event.import(force: true) - - search_query = ' - fragment facetFields on Facet { - id - title - count - } - query{ - works(query:"*"){ - totalCount - fieldsOfScience { ...facetFields } - fieldsOfScienceRepository { ...facetFields } - fieldsOfScienceCombined{ ...facetFields } - } - } - ' - - create(:prefix) - client = create(:client_with_fos) - create_list(:doi, WORK_COUNT, - aasm_state: "findable", - client: client - ) - Doi.import - sleep SLEEP_TIME - @facet_response = LupoSchema.execute(search_query).as_json - Rails.logger.level = :fatal - DataciteDoi.destroy_all - ReferenceRepository.destroy_all - Client.destroy_all - Provider.destroy_all - Prefix.destroy_all - ClientPrefix.destroy_all - ProviderPrefix.destroy_all - Event.destroy_all - end - - let (:fos_facet) do - { - "id" => "physical_sciences", - "title" => "Physical sciences", - "count" => WORK_COUNT - } - end - - it "has all dois in the search results" do - response = @facet_response - expect(response.dig("data", "works", "totalCount")).to eq(WORK_COUNT) - end - - it "returns Field of Science Facets" do - response = @facet_response - expect( - response.dig("data", "works", "fieldsOfScience") - ).to match_array([]) - end - - it "returns Field of Science Facets from the repository" do - response = @facet_response - expect( - response.dig("data", "works", "fieldsOfScienceRepository") - ).to match_array([ fos_facet ]) - end - - it "returns combined Field of Science Facets" do - response = @facet_response - expect( - response.dig("data", "works", "fieldsOfScienceCombined") - ).to match_array([ fos_facet ]) - end - end end diff --git a/spec/models/client_spec.rb b/spec/models/client_spec.rb index e9872b420..921389037 100644 --- a/spec/models/client_spec.rb +++ b/spec/models/client_spec.rb @@ -15,11 +15,7 @@ end describe "subjects" do - let(:client) { build(:client, - provider: provider, - repository_type: "disciplinary" - ) - } + let(:client) { build(:client, provider: provider) } let(:valid_subjects) do { classificationCode: "6.1", @@ -29,12 +25,6 @@ } end - it "are invalid if the repository_type is not disciplinary" do - client.repository_type = nil - client.subjects = [valid_subjects] - expect(client).to_not be_valid - end - it "valid hash" do client.subjects = [valid_subjects] expect(client).to be_valid diff --git a/spec/requests/repositories_spec.rb b/spec/requests/repositories_spec.rb index 374841395..3f2512696 100644 --- a/spec/requests/repositories_spec.rb +++ b/spec/requests/repositories_spec.rb @@ -413,7 +413,6 @@ "type" => "repositories", "attributes" => { "subjects" => subjects, - "repositoryType" => "disciplinary", }, }, }