Support to send HTTP Response Security Event

lib/newrelic_security/agent/agent.rb

@@ -7,6 +7,8 @@
 require 'newrelic_security/agent/control/fuzz_request'
 require 'newrelic_security/agent/control/reflected_xss'
 require 'newrelic_security/agent/control/http_context'
+require 'newrelic_security/agent/control/http_response'
+require 'newrelic_security/agent/control/http_response_event'
 require 'newrelic_security/agent/control/grpc_context'
 require 'newrelic_security/agent/control/collector'
 require 'newrelic_security/agent/control/app_info'

lib/newrelic_security/agent/configuration/manager.rb

@@ -33,13 +33,14 @@ def initialize
           @cache[:'security.detection.rci.enabled'] = ::NewRelic::Agent.config[:'security.detection.rci.enabled']
           @cache[:'security.detection.rxss.enabled'] = ::NewRelic::Agent.config[:'security.detection.rxss.enabled']
           @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
+          @cache[:'security.scan_controllers.report_http_response_body'] = ::NewRelic::Agent.config[:'security.scan_controllers.report_http_response_body']
           @cache[:framework] = detect_framework
           @cache[:'security.application_info.port'] = ::NewRelic::Agent.config[:'security.application_info.port'].to_i
           @cache[:'security.request.body_limit'] = ::NewRelic::Agent.config[:'security.request.body_limit'].to_i > 0 ? ::NewRelic::Agent.config[:'security.request.body_limit'].to_i : 300
           @cache[:listen_port] = nil
           @cache[:app_root] = NewRelic::Security::Agent::Utils.app_root
           @cache[:jruby_objectspace_enabled] = false
-          @cache[:json_version] = :'1.2.4'
+          @cache[:json_version] = :'1.2.10'
 
           @environment_source = NewRelic::Security::Agent::Configuration::EnvironmentSource.new
           @server_source = NewRelic::Security::Agent::Configuration::ServerSource.new

lib/newrelic_security/agent/control/app_info.rb

@@ -50,7 +50,7 @@ def initialize
           @osVersion = os_version
           @serverInfo = Hash.new # TODO: Fill this
           @identifier = Hash.new # TODO: Fill this
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
         end
 
         def as_json
@@ -121,13 +121,6 @@ def os_version
           ::Gem::Platform.local.version
         end
 
-        def add_linking_metadata
-          linking_metadata = Hash.new
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-          # TODO: add other fields as well in linking metadata, for event and heathcheck as well
-        end
-
       end
     end
   end

lib/newrelic_security/agent/control/application_runtime_error.rb

@@ -25,7 +25,7 @@ def initialize(exception, ctxt, response_code, category)
           @framework = NewRelic::Security::Agent.config[:framework]
           @groupName = NewRelic::Security::Agent.config[:mode]
           @policyVersion = nil
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
           @httpRequest = get_http_request_data(ctxt)
           @exception = exception
           @counter = 1
@@ -50,13 +50,6 @@ def current_time_millis
           (Time.now.to_f * 1000).to_i
         end
 
-        def add_linking_metadata
-          linking_metadata = {}
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-          # TODO: add other fields as well in linking metadata, for event and heathcheck as well
-        end
-
         def get_http_request_data(ctxt)
           return if ctxt.nil?
           http_request = {}

lib/newrelic_security/agent/control/application_url_mappings.rb

@@ -26,7 +26,7 @@ def initialize
           @framework = NewRelic::Security::Agent.config[:framework]
           @groupName = NewRelic::Security::Agent.config[:mode]
           @policyVersion = nil
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
           @mappings = []
         end
 
@@ -36,7 +36,7 @@ def as_json
           end.to_h
         end
 
-        def to_json
+        def to_json # rubocop:disable Lint/ToJSON
           as_json.to_json
         end
 
@@ -55,13 +55,6 @@ def current_time_millis
           (Time.now.to_f * 1000).to_i
         end
 
-        def add_linking_metadata
-          linking_metadata = {}
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-          # TODO: add other fields as well in linking metadata, for event and heathcheck as well
-        end
-
       end
     end
   end

lib/newrelic_security/agent/control/critical_message.rb

@@ -23,7 +23,7 @@ def initialize(message, level, caller, thread_name, exception = nil)
           @applicationUUID = NewRelic::Security::Agent.config[:uuid]
           @appAccountId = NewRelic::Security::Agent.config[:account_id]
           @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
           @timestamp = current_time_millis
           @message = message
           @level = level          
@@ -38,7 +38,7 @@ def as_json
           end.to_h
         end
 
-        def to_json
+        def to_json # rubocop:disable Lint/ToJSON
           as_json.to_json
         end
 
@@ -48,12 +48,6 @@ def current_time_millis
           (Time.now.to_f * 1000).to_i
         end
 
-        def add_linking_metadata
-          linking_metadata = Hash.new
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-        end
-
       end
     end
   end

lib/newrelic_security/agent/control/event.rb

@@ -31,7 +31,7 @@ def initialize(case_type, args, event_category)
           @httpRequest = Hash.new
           @httpResponse = Hash.new
           @metaData = { :reflectedMetaData => { :listen_port => NewRelic::Security::Agent.config[:listen_port].to_s }, :appServerInfo => { :applicationDirectory => NewRelic::Security::Agent.config[:app_root], :serverBaseDirectory => NewRelic::Security::Agent.config[:app_root] } }
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
           @pid = pid
           @parameters = args
           @sourceMethod = nil
@@ -146,12 +146,6 @@ def thread_monotonic_ctr
           end
         end
 
-        def add_linking_metadata
-          linking_metadata = Hash.new
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-        end
-
       end
     end
   end

lib/newrelic_security/agent/control/event_processor.rb

@@ -53,6 +53,11 @@ def send_event(event)
           event = nil
         end
 
+        def send_http_response_event(http_response_event)
+          enqueue(http_response_event)
+          http_response_event = nil
+        end
+
         def send_health
           health = NewRelic::Security::Agent::Control::Health.new
           health.update_health_check

lib/newrelic_security/agent/control/health_check.rb

@@ -29,7 +29,7 @@ def initialize
           @httpRequestCount = 0
           @protectedVulnerabilties = nil
           @protectedDB = nil
-          @linkingMetadata = add_linking_metadata
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
           @stats = {}
           @serviceStatus = {} # TODO: Fill this
           @iastEventStats = {}
@@ -80,12 +80,6 @@ def current_time_millis
           (Time.now.to_f * 1000).to_i
         end
 
-        def add_linking_metadata
-          linking_metadata = Hash.new
-          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
-          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
-        end
-
         def system_total_memory_mb
           case RbConfig::CONFIG['host_os']
           when /darwin9/

lib/newrelic_security/agent/control/http_context.rb

@@ -18,8 +18,9 @@ module Control
       class HTTPContext
 
         attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter, :mutex
+        attr_reader :trace_id, :span_id
 
-        def initialize(env)
+        def initialize(env, trace_id, span_id)
           @time_stamp = current_time_millis
           @req = env.select { |key, _| CGI_VARIABLES.include? key}
           @method = @req[REQUEST_METHOD]
@@ -49,6 +50,8 @@ def initialize(env)
           @fuzz_files = ::Set.new
           @event_counter = 0
           @mutex = Mutex.new
+          @trace_id = trace_id
+          @span_id = span_id
           NewRelic::Security::Agent.agent.http_request_count.increment
           NewRelic::Security::Agent.agent.iast_client.completed_requests[@headers[NR_CSEC_PARENT_ID]] = [] if @headers.key?(NR_CSEC_PARENT_ID)
         end
@@ -62,7 +65,8 @@ def self.get_context
         end
 
         def self.set_context(env)
-          ::NewRelic::Agent::Tracer.current_transaction.instance_variable_set(:@security_context_data, HTTPContext.new(env))
+          current_transaction = get_current_transaction
+          current_transaction.instance_variable_set(:@security_context_data, HTTPContext.new(env, current_transaction.trace_id, current_transaction.current_segment.guid))
         end
 
         def self.reset_context

lib/newrelic_security/agent/control/http_response.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module NewRelic::Security
+  module Agent
+    module Control
+
+      class HTTPResponse
+
+        attr_accessor :statusCode, :headers, :body
+
+        def initialize(status, headers, body)
+          @statusCode = status
+          @headers = headers
+          @body = read_body_to_string(body)
+          @contentType = headers[Content_Type]
+        end
+
+        def as_json
+          instance_variables.map! do |ivar|
+            [ivar[1..-1].to_sym, instance_variable_get(ivar)]
+          end.to_h
+        end
+
+        def to_json # rubocop:disable Lint/ToJSON
+          as_json.to_json
+        end
+
+        private
+
+        def read_body_to_string(body)
+          response_body = ::String.new
+          body.each { |string| response_body << string }
+          response_body
+        end
+      end
+    end
+  end
+end

lib/newrelic_security/agent/control/http_response_event.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module NewRelic::Security
+  module Agent
+    module Control
+
+      class HTTPResponseEvent
+
+        attr_accessor :isIASTRequest, :httpRequest, :httpResponse
+        attr_reader :jsonName
+
+        def initialize(ctxt, http_response)
+          @collectorType = RUBY
+          @language = Ruby
+          @jsonName = :sec_http_response
+          @eventType = :sec_http_response
+          @framework = NewRelic::Security::Agent.config[:framework]
+          @groupName = NewRelic::Security::Agent.config[:mode]
+          @policyVersion = nil
+          @collectorVersion = NewRelic::Security::VERSION
+          @buildNumber = nil
+          @jsonVersion = NewRelic::Security::Agent.config[:json_version]
+          @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
+          @httpRequest = {}
+          @httpResponse = http_response.as_json
+          @linkingMetadata = NewRelic::Security::Agent::Utils.add_linking_metadata
+          @traceId = ctxt.trace_id
+          @isIASTRequest = NewRelic::Security::Agent::Utils.is_IAST_request?(ctxt.headers)
+        end
+
+        def as_json
+          instance_variables.map! do |ivar|
+            [ivar[1..-1].to_sym, instance_variable_get(ivar)]
+          end.to_h
+        end
+
+        def to_json # rubocop:disable Lint/ToJSON
+          as_json.to_json
+        end
+
+      end
+    end
+  end
+end

lib/newrelic_security/agent/control/reflected_xss.rb

@@ -15,7 +15,6 @@ module ReflectedXSS
         HTML_COMMENT_END = '-->'
         FIVE_COLON = ':::::'
         SCRIPT = 'script'
-        Content_Type = 'Content-Type'
         QUERY_STRING = 'QUERY_STRING'
         REQUEST_URI = 'REQUEST_URI'
         APPLICATION_JSON = 'application/json'
@@ -43,21 +42,19 @@ module ReflectedXSS
 
         extend self
 
-        def check_xss(http_req, retval)
+        def check_xss(http_req, http_response)
           # TODO: Check if enableHTTPRequestPrinting is required.
-          return if http_req.nil? || retval.empty?
-          if retval[1].key?(Content_Type) && (retval[1][Content_Type].start_with?(*UNSUPPORTED_MEDIA_TYPES) || retval[1][Content_Type].start_with?(*UNSUPPORTED_CONTENT_TYPES))
+          return if http_req.nil? || http_response.nil?
+          if http_response.headers.key?(Content_Type) && (http_response.headers[Content_Type].start_with?(*UNSUPPORTED_MEDIA_TYPES) || http_response.headers[Content_Type].start_with?(*UNSUPPORTED_CONTENT_TYPES))
             return
           end
-          response_body = ::String.new
-          retval[2].each { |string| response_body << string }
-          construct = check_for_reflected_xss(http_req, retval[1], response_body)
+          construct = check_for_reflected_xss(http_req, http_response.headers, http_response.body)
           NewRelic::Security::Agent.logger.debug "RXSS Attack DATA: #{construct}"
           if !construct.empty? || NewRelic::Security::Agent::Utils.is_IAST?
             parameters = Array.new
             parameters << construct
-            parameters << response_body.force_encoding(ISO_8859_1).encode(UTF_8)
-            NewRelic::Security::Agent::Control::Collector.collect(REFLECTED_XSS, parameters, nil, :response_header => retval[1][Content_Type])
+            parameters << http_response.body.force_encoding(ISO_8859_1).encode(UTF_8)
+            NewRelic::Security::Agent::Control::Collector.collect(REFLECTED_XSS, parameters, nil, :response_header => http_response.headers[Content_Type])
           end
         rescue Exception => exception
           NewRelic::Security::Agent.logger.error "Exception in Reflected XSS detection : #{exception.inspect} #{exception.backtrace}"

lib/newrelic_security/agent/utils/agent_utils.rb

@@ -9,7 +9,6 @@ module Utils
       extend self
 
       ENABLED = 'enabled'
-      VULNERABLE = 'VULNERABLE'
       AES_256_CBC = 'AES-256-CBC'
       H_ASTERIK = 'H*'
       ASTERISK = '*'
@@ -25,6 +24,16 @@ def is_IAST_request?(headers)
         false
       end
 
+      def add_linking_metadata
+        linking_metadata = {}
+        linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
+        if (ctxt = NewRelic::Security::Agent::Control::HTTPContext.get_context)
+          linking_metadata[:'trace.id'] = ctxt.trace_id
+          linking_metadata[:'span.id'] = ctxt.span_id
+        end
+        linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
+      end
+
       def parse_fuzz_header(ctxt)
         headers = ctxt.headers if ctxt
         if is_IAST? && is_IAST_request?(headers)

lib/newrelic_security/constants.rb

@@ -15,6 +15,7 @@ module NewRelic::Security
   LOG_FILE_NAME = 'ruby-security-collector.log'
   NR_SECURITY_HOME_TMP = 'nr-security-home/tmp/'
   NR_CSEC_FUZZ_REQUEST_ID = 'nr-csec-fuzz-request-id'
+  VULNERABLE = 'VULNERABLE'
   NR_CSEC_TRACING_DATA = 'nr-csec-tracing-data'
   NR_CSEC_PARENT_ID = 'nr-csec-parent-id'
   COLON_IAST_COLON = ':IAST:'
@@ -55,6 +56,7 @@ module NewRelic::Security
   REQUEST_METHOD = 'REQUEST_METHOD'
   PATH_INFO = 'PATH_INFO'
   CONTENT_TYPE = 'CONTENT_TYPE'
+  Content_Type = 'Content-Type'
   REQUEST_URI = 'REQUEST_URI'
   SERVER_PORT = 'SERVER_PORT'
   X_FORWARDED_FOR = 'x-forwarded-for'