From 8f512572fc04023feb7cf25e5232609a3c8620b7 Mon Sep 17 00:00:00 2001 From: kjgarza Date: Fri, 7 Feb 2020 19:00:54 +0100 Subject: [PATCH 1/4] moved lograge configuration back into application. already confirmed that changes wont take effect from the initilizer --- config/initializers/_lograge.rb | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 config/initializers/_lograge.rb diff --git a/config/initializers/_lograge.rb b/config/initializers/_lograge.rb deleted file mode 100644 index 4065e2caa..000000000 --- a/config/initializers/_lograge.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require "elasticsearch/rails/lograge" - -Rails.application.configure do - config.lograge.enabled = true - config.lograge.formatter = Lograge::Formatters::Logstash.new - config.lograge.logger = LogStashLogger.new(type: :stdout) - config.logger = config.lograge.logger - config.lograge.log_level = ENV["LOG_LEVEL"].to_sym - config.logger = config.lograge.logger - - config.lograge.ignore_actions = ["HeartbeatController#index", "IndexController#index"] - config.lograge.ignore_custom = lambda do |event| - event.payload.inspect.length > 100000 - end - config.lograge.base_controller_class = "ActionController::API" - - config.lograge.custom_options = lambda do |event| - exceptions = %w(controller action format id) - { - params: event.payload[:params].except(*exceptions), - uid: event.payload[:uid], - } - end -end - -Rails.cache.silence! From 46229330aa9f91b1a8f22083efbd9c8f6f070dad Mon Sep 17 00:00:00 2001 From: kjgarza Date: Fri, 7 Feb 2020 19:02:19 +0100 Subject: [PATCH 2/4] move cache silencer to enviroment.rb --- config/environment.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/environment.rb b/config/environment.rb index 426333bb4..b99536038 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -3,3 +3,4 @@ # Initialize the Rails application. Rails.application.initialize! +Rails.cache.silence! \ No newline at end of file From 1877a09e5051ad7315f8db41b31b98646d92f0f5 Mon Sep 17 00:00:00 2001 From: kjgarza Date: Fri, 7 Feb 2020 19:05:25 +0100 Subject: [PATCH 3/4] set back logging to november position and correctly configure log_level ## LogStashLogger needs to be pass to rails logger, see roidrage/lograge#26 ## Log level in a config level configuration Confirmed that log_level is respected --- config/application.rb | 44 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/config/application.rb b/config/application.rb index 1a9a783ff..f8b1fca29 100644 --- a/config/application.rb +++ b/config/application.rb @@ -83,34 +83,30 @@ class Application < Rails::Application # secret_key_base is not used by Rails API, as there are no sessions config.secret_key_base = "blipblapblup" - # config.lograge.enabled = true - # config.lograge.formatter = Lograge::Formatters::Logstash.new - # config.lograge.logger = ::LogStashLogger.new( - # type: :stdout, - # ) - # config.lograge.log_level = ENV["LOG_LEVEL"].to_sym - - # config.active_job.logger = config.lograge.logger - - # config.lograge.ignore_actions = ["HeartbeatController#index", "IndexController#index"] - # config.lograge.ignore_custom = lambda do |event| - # event.payload.inspect.length > 100000 - # end - # config.lograge.base_controller_class = "ActionController::API" - - # config.lograge.custom_options = lambda do |event| - # exceptions = %w(controller action format id) - # { - # params: event.payload[:params].except(*exceptions), - # uid: event.payload[:uid], - # } - # end - # config.logger = config.lograge.logger - # Disable loggers that log to file config.active_record.logger = nil config.active_job.logger = nil + config.lograge.enabled = true + config.lograge.formatter = Lograge::Formatters::Logstash.new + config.lograge.logger = LogStashLogger.new(type: :stdout) + config.logger = config.lograge.logger ## LogStashLogger needs to be pass to rails logger, see roidrage/lograge#26 + config.log_level = ENV["LOG_LEVEL"].to_sym ## Log level in a config level configuration + + config.lograge.ignore_actions = ["HeartbeatController#index", "IndexController#index"] + config.lograge.ignore_custom = lambda do |event| + event.payload.inspect.length > 100000 + end + config.lograge.base_controller_class = "ActionController::API" + + config.lograge.custom_options = lambda do |event| + exceptions = %w(controller action format id) + { + params: event.payload[:params].except(*exceptions), + uid: event.payload[:uid], + } + end + # configure caching config.cache_store = :dalli_store, nil, { :namespace => ENV['APPLICATION'] } From 2de18cd8540c871a875ff2efda4061b22bbb8657 Mon Sep 17 00:00:00 2001 From: Martin Fenner Date: Fri, 7 Feb 2020 20:06:25 +0100 Subject: [PATCH 4/4] minor tweaks --- app/controllers/dois_controller.rb | 4 +--- app/controllers/events_controller.rb | 14 ++++++-------- app/controllers/works_controller.rb | 19 +++++++------------ app/models/doi.rb | 8 +++----- app/models/event.rb | 9 ++++----- config/application.rb | 6 +----- 6 files changed, 22 insertions(+), 38 deletions(-) diff --git a/app/controllers/dois_controller.rb b/app/controllers/dois_controller.rb index 57656beff..a3569b8dc 100644 --- a/app/controllers/dois_controller.rb +++ b/app/controllers/dois_controller.rb @@ -32,8 +32,6 @@ def index else { updated: { order: 'desc' }} end - logger = LogStashLogger.new(type: :stdout) - page = page_from_params(params) sample_group_field = case params[:sample_group] @@ -88,7 +86,7 @@ def index random: params[:random]) end } - logger.warn method: "GET", path: "/dois", message: "Request /dois", duration: bm + Rails.logger.warn method: "GET", path: "/dois", message: "Request /dois", duration: bm begin # If we're using sample groups we need to unpack the results from the aggregation bucket hits. diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 861043a70..253c514c4 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -66,8 +66,6 @@ def show end def index - logger = LogStashLogger.new(type: :stdout) - sort = case params[:sort] when "relevance" then { "_score" => { order: 'desc' }} when "obj_id" then { "obj_id" => { order: 'asc' }} @@ -113,7 +111,7 @@ def index sort: sort) end } - logger.warn method: "GET", path: "/events", message: "Request /events", duration: bm + Rails.logger.warn method: "GET", path: "/events", message: "Request /events", duration: bm if page[:scroll].present? results = response.results @@ -184,7 +182,7 @@ def index unique_obj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_obj_count.value : nil unique_subj_count = total.positive? && aggregations.include?("advanced_aggregations") ? response.response.aggregations.unique_subj_count.value : nil } - logger.warn method: "GET", path: "/events", message: "Aggregations /events", duration: bm + Rails.logger.warn method: "GET", path: "/events", message: "Aggregations /events", duration: bm results = response.results @@ -244,8 +242,8 @@ def index bm = Benchmark.ms { events_serialized = EventSerializer.new(results, options).serializable_hash } - logger.warn method: "GET", path: "/events", message: "Serialize /events", duration: bm - + Rails.logger.warn method: "GET", path: "/events", message: "Serialize /events", duration: bm + if @include.include?(:dois) doi_names = "" bm = Benchmark.ms { @@ -253,13 +251,13 @@ def index doi_names = (results.map { |event| event.doi}).uniq().join(",") events_serialized[:included] = DoiSerializer.new((Doi.find_by_id(doi_names).results), is_collection: true).serializable_hash.dig(:data) } - logger.warn method: "GET", path: "/events", message: "IncludeDois /events", duration: bm + Rails.logger.warn method: "GET", path: "/events", message: "IncludeDois /events", duration: bm end bm = Benchmark.ms { render json: events_serialized, status: :ok } - logger.warn method: "GET", path: "/events", message: "Render /events", duration: bm + Rails.logger.warn method: "GET", path: "/events", message: "Render /events", duration: bm end end diff --git a/app/controllers/works_controller.rb b/app/controllers/works_controller.rb index 8ad8db09c..09d7e416c 100644 --- a/app/controllers/works_controller.rb +++ b/app/controllers/works_controller.rb @@ -16,8 +16,6 @@ def index else { updated: { order: 'desc' }} end - logger = LogStashLogger.new(type: :stdout) - page = page_from_params(params) sample_group_field = case params[:sample_group] @@ -54,7 +52,7 @@ def index random: params[:sample].present? ? true : false) end } - logger.warn method: "GET", path: "/works", message: "Request /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "Request /works", duration: bm begin total = response.results.total @@ -72,12 +70,12 @@ def index providers = total > 0 ? facet_by_provider(response.response.aggregations.providers.buckets) : nil clients = total > 0 ? facet_by_client(response.response.aggregations.clients.buckets) : nil } - logger.warn method: "GET", path: "/works", message: "AggregationsBasic /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "AggregationsBasic /works", duration: bm bm = Benchmark.ms { affiliations = total > 0 ? facet_by_affiliation(response.response.aggregations.affiliations.buckets) : nil } - logger.warn method: "GET", path: "/works", message: "AggregationsAffiliations /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "AggregationsAffiliations /works", duration: bm @dois = response.results @@ -119,7 +117,7 @@ def index bm = Benchmark.ms { render json: WorkSerializer.new(@dois, options).serialized_json, status: :ok } - logger.warn method: "GET", path: "/works", message: "Render /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "Render /works", duration: bm rescue Elasticsearch::Transport::Transport::Errors::BadRequest => exception message = JSON.parse(exception.message[6..-1]).to_h.dig("error", "root_cause", 0, "reason") @@ -139,22 +137,19 @@ def show bm = Benchmark.ms { render json: WorkSerializer.new(@doi, options).serialized_json, status: :ok } - logger = LogStashLogger.new(type: :stdout) - logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Render /works/#{@doi.doi}", duration: bm + Rails.logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Render /works/#{@doi.doi}", duration: bm end protected def set_doi - logger = LogStashLogger.new(type: :stdout) - @doi = nil - + bm = Benchmark.ms { @doi = Doi.where(doi: params[:id], aasm_state: "findable").first } fail ActiveRecord::RecordNotFound if @doi.blank? - logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Request DB /works/#{@doi.doi}", duration: bm + Rails.logger.warn method: "GET", path: "/works/#{@doi.doi}", message: "Request DB /works/#{@doi.doi}", duration: bm end def set_include diff --git a/app/models/doi.rb b/app/models/doi.rb index d108eb6dc..6ba02581a 100644 --- a/app/models/doi.rb +++ b/app/models/doi.rb @@ -610,8 +610,6 @@ def self.find_by_id(ids, options={}) end def self.query(query, options={}) - logger = LogStashLogger.new(type: :stdout) - # support scroll api # map function is small performance hit if options[:scroll_id].present? && options.dig(:page, :scroll) @@ -639,7 +637,7 @@ def self.query(query, options={}) bm = Benchmark.ms { aggregations = get_aggregations_hash(options) } - logger.warn method: "GET", path: "/works", message: "QueryAggregations /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "QueryAggregations /works", duration: bm options[:page] ||= {} options[:page][:number] ||= 1 @@ -763,7 +761,7 @@ def self.query(query, options={}) } end } - logger.warn method: "GET", path: "/works", message: "QueryProcessing /works", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "QueryProcessing /works", duration: bm # three options for going through results are scroll, cursor and pagination # the default is pagination @@ -810,7 +808,7 @@ def self.query(query, options={}) }.compact) end } - logger.warn method: "GET", path: "/works", message: "Query /works #{es_query.inspect}", duration: bm + Rails.logger.warn method: "GET", path: "/works", message: "Query /works #{es_query.inspect}", duration: bm response end diff --git a/app/models/event.rb b/app/models/event.rb index 19228415f..6c7db8c8a 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -533,14 +533,13 @@ def access_method end def self.subj_id_check(options = {}) - logger = LogStashLogger.new(type: :stdout) file_name = "evens_with_double_crossref_dois.txt" size = (options[:size] || 1000).to_i cursor = [options[:from_id], options[:until_id]] total_errors = 0 response = Event.query(nil, source_id: "datacite-crossref", page: { size: 1, cursor: [] }) - logger.warn "[DoubleCheck] #{response.results.total} events for source datacite-crossref." + Rails.logger.warn "[DoubleCheck] #{response.results.total} events for source datacite-crossref." # walk through results using cursor if response.results.total.positive? @@ -548,7 +547,7 @@ def self.subj_id_check(options = {}) response = Event.query(nil, source_id: "datacite-crossref", page: { size: size, cursor: cursor }) break unless response.results.results.length.positive? - logger.warn "[DoubleCheck] DoubleCheck #{response.results.results.length} events starting with _id #{response.results.to_a.first[:_id]}." + Rails.logger.warn "[DoubleCheck] DoubleCheck #{response.results.results.length} events starting with _id #{response.results.to_a.first[:_id]}." cursor = response.results.to_a.last[:sort] # dois = response.results.results.map(&:subj_id) @@ -570,8 +569,8 @@ def self.subj_id_check(options = {}) payload = { description: "events_with_errors_from_rake_task #{Time.now.getutc}", public: true,files: {uids_with_errors: {content: file.read} }} ### max file size 1MB response = Maremma.post("https://api.github.com/gists", data: payload.to_json, username: ENV["GIST_USERNAME"], password:ENV["GIST_PASSWORD"]) - logger.warn "[DoubleCheck] Total number of events with Errors: #{total_errors}" - logger.warn "[DoubleCheck] IDs saved: #{response.body.dig('data','url')}" if [200,201].include?(response.status) + Rails.logger.warn "[DoubleCheck] Total number of events with Errors: #{total_errors}" + Rails.logger.warn "[DoubleCheck] IDs saved: #{response.body.dig('data','url')}" if [200,201].include?(response.status) end end diff --git a/config/application.rb b/config/application.rb index f8b1fca29..24b8b83fe 100644 --- a/config/application.rb +++ b/config/application.rb @@ -83,13 +83,9 @@ class Application < Rails::Application # secret_key_base is not used by Rails API, as there are no sessions config.secret_key_base = "blipblapblup" - # Disable loggers that log to file - config.active_record.logger = nil - config.active_job.logger = nil - config.lograge.enabled = true config.lograge.formatter = Lograge::Formatters::Logstash.new - config.lograge.logger = LogStashLogger.new(type: :stdout) + config.lograge.logger = LogStashLogger.new(type: :stdout) config.logger = config.lograge.logger ## LogStashLogger needs to be pass to rails logger, see roidrage/lograge#26 config.log_level = ENV["LOG_LEVEL"].to_sym ## Log level in a config level configuration