Skip to content

Commit

Permalink
minor tweaks
Browse files Browse the repository at this point in the history
  • Loading branch information
Martin Fenner committed Feb 7, 2020
1 parent 1877a09 commit 2de18cd
Show file tree
Hide file tree
Showing 6 changed files with 22 additions and 38 deletions.
4 changes: 1 addition & 3 deletions app/controllers/dois_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ def index
else { updated: { order: 'desc' }}
end

logger = LogStashLogger.new(type: :stdout)

page = page_from_params(params)

sample_group_field = case params[:sample_group]
Expand Down Expand Up @@ -88,7 +86,7 @@ def index
random: params[:random])
end
}
logger.warn method: "GET", path: "/dois", message: "Request /dois", duration: bm
Rails.logger.warn method: "GET", path: "/dois", message: "Request /dois", duration: bm

begin
# If we're using sample groups we need to unpack the results from the aggregation bucket hits.
Expand Down
14 changes: 6 additions & 8 deletions app/controllers/events_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,6 @@ def show
end

def index
logger = LogStashLogger.new(type: :stdout)

sort = case params[:sort]
when "relevance" then { "_score" => { order: 'desc' }}
when "obj_id" then { "obj_id" => { order: 'asc' }}
Expand Down Expand Up @@ -113,7 +111,7 @@ def index
sort: sort)
end
}
logger.warn method: "GET", path: "/events", message: "Request /events", duration: bm
Rails.logger.warn method: "GET", path: "/events", message: "Request /events", duration: bm

if page[:scroll].present?
results = response.results
Expand Down Expand Up @@ -184,7 +182,7 @@ def index
unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil
unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil
}
logger.warn method: "GET", path: "/events", message: "Aggregations /events", duration: bm
Rails.logger.warn method: "GET", path: "/events", message: "Aggregations /events", duration: bm

results = response.results

Expand Down Expand Up @@ -244,22 +242,22 @@ def index
bm = Benchmark.ms {
events_serialized = EventSerializer.new(results, options).serializable_hash
}
logger.warn method: "GET", path: "/events", message: "Serialize /events", duration: bm
Rails.logger.warn method: "GET", path: "/events", message: "Serialize /events", duration: bm

if @include.include?(:dois)
doi_names = ""
bm = Benchmark.ms {
options[:include] = []
doi_names = (results.map { |event| event.doi}).uniq().join(",")
events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), is_collection: true).serializable_hash.dig(:data)
}
logger.warn method: "GET", path: "/events", message: "IncludeDois /events", duration: bm
Rails.logger.warn method: "GET", path: "/events", message: "IncludeDois /events", duration: bm
end

bm = Benchmark.ms {
render json: events_serialized, status: :ok
}
logger.warn method: "GET", path: "/events", message: "Render /events", duration: bm
Rails.logger.warn method: "GET", path: "/events", message: "Render /events", duration: bm
end
end

Expand Down
19 changes: 7 additions & 12 deletions app/controllers/works_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ def index
else { updated: { order: 'desc' }}
end

logger = LogStashLogger.new(type: :stdout)

page = page_from_params(params)

sample_group_field = case params[:sample_group]
Expand Down Expand Up @@ -54,7 +52,7 @@ def index
random: params[:sample].present? ? true : false)
end
}
logger.warn method: "GET", path: "/works", message: "Request /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "Request /works", duration: bm

begin
total = response.results.total
Expand All @@ -72,12 +70,12 @@ def index
providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil
clients = total > 0 ? facet_by_client(response.response.aggregations.clients.buckets) : nil
}
logger.warn method: "GET", path: "/works", message: "AggregationsBasic /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "AggregationsBasic /works", duration: bm

bm = Benchmark.ms {
affiliations = total > 0 ? facet_by_affiliation(response.response.aggregations.affiliations.buckets) : nil
}
logger.warn method: "GET", path: "/works", message: "AggregationsAffiliations /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "AggregationsAffiliations /works", duration: bm

@dois = response.results

Expand Down Expand Up @@ -119,7 +117,7 @@ def index
bm = Benchmark.ms {
render json: WorkSerializer.new(@dois, options).serialized_json, status: :ok
}
logger.warn method: "GET", path: "/works", message: "Render /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "Render /works", duration: bm
rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception
message = JSON.parse(exception.message[6..-1]).to_h.dig("error", "root_cause", 0, "reason")

Expand All @@ -139,22 +137,19 @@ def show
bm = Benchmark.ms {
render json: WorkSerializer.new(@doi, options).serialized_json, status: :ok
}
logger = LogStashLogger.new(type: :stdout)
logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Render /works/#{@doi.doi}", duration: bm
Rails.logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Render /works/#{@doi.doi}", duration: bm
end

protected

def set_doi
logger = LogStashLogger.new(type: :stdout)

@doi = nil

bm = Benchmark.ms {
@doi = Doi.where(doi: params[:id], aasm_state: "findable").first
}
fail ActiveRecord::RecordNotFound if @doi.blank?
logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Request DB /works/#{@doi.doi}", duration: bm
Rails.logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Request DB /works/#{@doi.doi}", duration: bm
end

def set_include
Expand Down
8 changes: 3 additions & 5 deletions app/models/doi.rb
Original file line number Diff line number Diff line change
Expand Up @@ -610,8 +610,6 @@ def self.find_by_id(ids, options={})
end

def self.query(query, options={})
logger = LogStashLogger.new(type: :stdout)

# support scroll api
# map function is small performance hit
if options[:scroll_id].present? && options.dig(:page, :scroll)
Expand Down Expand Up @@ -639,7 +637,7 @@ def self.query(query, options={})
bm = Benchmark.ms {
aggregations = get_aggregations_hash(options)
}
logger.warn method: "GET", path: "/works", message: "QueryAggregations /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "QueryAggregations /works", duration: bm

options[:page] ||= {}
options[:page][:number] ||= 1
Expand Down Expand Up @@ -763,7 +761,7 @@ def self.query(query, options={})
}
end
}
logger.warn method: "GET", path: "/works", message: "QueryProcessing /works", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "QueryProcessing /works", duration: bm

# three options for going through results are scroll, cursor and pagination
# the default is pagination
Expand Down Expand Up @@ -810,7 +808,7 @@ def self.query(query, options={})
}.compact)
end
}
logger.warn method: "GET", path: "/works", message: "Query /works #{es_query.inspect}", duration: bm
Rails.logger.warn method: "GET", path: "/works", message: "Query /works #{es_query.inspect}", duration: bm
response
end

Expand Down
9 changes: 4 additions & 5 deletions app/models/event.rb
Original file line number Diff line number Diff line change
Expand Up @@ -533,22 +533,21 @@ def access_method
end

def self.subj_id_check(options = {})
logger = LogStashLogger.new(type: :stdout)
file_name = "evens_with_double_crossref_dois.txt"
size = (options[:size] || 1000).to_i
cursor = [options[:from_id], options[:until_id]]
total_errors = 0

response = Event.query(nil, source_id: "datacite-crossref", page: { size: 1, cursor: [] })
logger.warn "[DoubleCheck] #{response.results.total} events for source datacite-crossref."
Rails.logger.warn "[DoubleCheck] #{response.results.total} events for source datacite-crossref."

# walk through results using cursor
if response.results.total.positive?
while response.results.results.length.positive?
response = Event.query(nil, source_id: "datacite-crossref", page: { size: size, cursor: cursor })
break unless response.results.results.length.positive?

logger.warn "[DoubleCheck] DoubleCheck #{response.results.results.length} events starting with _id #{response.results.to_a.first[:_id]}."
Rails.logger.warn "[DoubleCheck] DoubleCheck #{response.results.results.length} events starting with _id #{response.results.to_a.first[:_id]}."
cursor = response.results.to_a.last[:sort]

# dois = response.results.results.map(&:subj_id)
Expand All @@ -570,8 +569,8 @@ def self.subj_id_check(options = {})
payload = { description: "events_with_errors_from_rake_task #{Time.now.getutc}", public: true,files: {uids_with_errors: {content: file.read} }}
### max file size 1MB
response = Maremma.post("https://api.github.com/gists", data: payload.to_json, username: ENV["GIST_USERNAME"], password:ENV["GIST_PASSWORD"])
logger.warn "[DoubleCheck] Total number of events with Errors: #{total_errors}"
logger.warn "[DoubleCheck] IDs saved: #{response.body.dig('data','url')}" if [200,201].include?(response.status)
Rails.logger.warn "[DoubleCheck] Total number of events with Errors: #{total_errors}"
Rails.logger.warn "[DoubleCheck] IDs saved: #{response.body.dig('data','url')}" if [200,201].include?(response.status)
end
end

Expand Down
6 changes: 1 addition & 5 deletions config/application.rb
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,9 @@ class Application < Rails::Application
# secret_key_base is not used by Rails API, as there are no sessions
config.secret_key_base = "blipblapblup"

# Disable loggers that log to file
config.active_record.logger = nil
config.active_job.logger = nil

config.lograge.enabled = true
config.lograge.formatter = Lograge::Formatters::Logstash.new
config.lograge.logger = LogStashLogger.new(type: :stdout)
config.lograge.logger = LogStashLogger.new(type: :stdout)
config.logger = config.lograge.logger ## LogStashLogger needs to be pass to rails logger, see roidrage/lograge#26
config.log_level = ENV["LOG_LEVEL"].to_sym ## Log level in a config level configuration

Expand Down

0 comments on commit 2de18cd

Please sign in to comment.