From 898cebe7e1af4ea59ccdf3672d76e3a1c8b23d04 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 17 Oct 2024 09:22:46 -0700 Subject: [PATCH 01/14] Add logging label forwarding implementation Finalize implementation of logging labels TEMP --- newrelic/config.py | 6 ++++++ newrelic/core/config.py | 15 ++++++++++++++- newrelic/core/data_collector.py | 21 ++++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 6a1ba2c69..12884dac0 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -210,6 +210,10 @@ def _map_inc_excl_attributes(s): return newrelic.core.config._parse_attributes(s) +def _map_case_insensitive_excl_labels(s): + return [v.lower() for v in newrelic.core.config._parse_attributes(s)] + + def _map_default_host_value(license_key): # If the license key is region aware, we should override the default host # to be the region aware host @@ -542,6 +546,8 @@ def _process_configuration(section): _process_setting(section, "application_logging.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.max_samples_stored", "getint", None) _process_setting(section, "application_logging.forwarding.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.include_labels.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.include_labels.exclude", "get", _map_case_insensitive_excl_labels) _process_setting(section, "application_logging.forwarding.context_data.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.context_data.include", "get", _map_inc_excl_attributes) _process_setting(section, "application_logging.forwarding.context_data.exclude", "get", _map_inc_excl_attributes) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3f5f213ee..66857ce98 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -323,6 +323,10 @@ class ApplicationLoggingForwardingSettings(Settings): pass +class ApplicationLoggingForwardingIncludeLabelsSettings(Settings): + pass + + class ApplicationLoggingForwardingContextDataSettings(Settings): pass @@ -424,6 +428,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.agent_limits = AgentLimitsSettings() _settings.application_logging = ApplicationLoggingSettings() _settings.application_logging.forwarding = ApplicationLoggingForwardingSettings() +_settings.application_logging.forwarding.include_labels = ApplicationLoggingForwardingIncludeLabelsSettings() _settings.application_logging.forwarding.context_data = ApplicationLoggingForwardingContextDataSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() @@ -935,6 +940,14 @@ def default_otlp_host(host): _settings.application_logging.forwarding.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=True ) + +_settings.application_logging.forwarding.include_labels.enabled = _environ_as_bool( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_INCLUDE_LABELS_ENABLED", default=False +) +_settings.application_logging.forwarding.include_labels.exclude = set( + v.lower() for v in _environ_as_set("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_INCLUDE_LABELS_EXCLUDE", default="") +) + _settings.application_logging.forwarding.context_data.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_ENABLED", default=False ) @@ -1096,7 +1109,7 @@ def global_settings_dump(settings_object=None, serializable=False): if not isinstance(key, str): del settings[key] - if not isinstance(value, str) and not isinstance(value, float) and not isinstance(value, int): + if not isinstance(value, (str, float, int)): settings[key] = repr(value) return settings diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 2d312bc0d..2f81fa529 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -35,7 +35,7 @@ _logger = logging.getLogger(__name__) -class Session(): +class Session: PROTOCOL = AgentProtocol OTLP_PROTOCOL = OtlpProtocol CLIENT = ApplicationModeClient @@ -154,10 +154,29 @@ def send_dimensional_metric_data(self, start_time, end_time, metric_data): payload = encode_metric_data(metric_data, start_time, end_time) return self._otlp_protocol.send("dimensional_metric_data", payload, path="/v1/metrics") + def get_log_events_labels(self): + """ "Generate common block for log events.""" + if not self.configuration.application_logging.forwarding.include_labels.enabled: + return {} + elif not self.configuration.application_logging.forwarding.include_labels.exclude: + return self.configuration.labels or {} + else: + return { + f"tags.{label['label_type']}": label['label_value'] + for label in self.configuration.labels + if label['label_type'].lower() not in self.configuration.application_logging.forwarding.include_labels.exclude + } + def send_log_events(self, sampling_info, log_event_data): """Called to submit sample set for log events.""" payload = ({"logs": tuple(log._asdict() for log in log_event_data)},) + + # Add labels into common block if enabled and not empty + labels = self.get_log_events_labels() + if labels: + payload[0]["common"] = {"attributes": labels} + return self._protocol.send("log_event_data", payload) def get_agent_commands(self): From da2bf6673538462e46ca6a973a4345ff7e816b4e Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 17 Oct 2024 10:24:58 -0700 Subject: [PATCH 02/14] Add tests for logging labels --- tests/agent_features/test_log_events.py | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index 9a619d8de..43a13a80b 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -396,3 +396,39 @@ def test(): record_log_event("A") test() + + +# Test Log Event Label Settings + + +# Add labels setting value in already processed format +TEST_LABELS = {"testlabel1": "A", "testlabel2": "B", "testlabelexclude": "C"} +TEST_LABELS = [{"label_type": k, "label_value": v} for k, v in TEST_LABELS.items()] + +@override_application_settings({ + "labels": TEST_LABELS, + "application_logging.forwarding.include_labels.enabled": True, + "application_logging.forwarding.include_labels.exclude": {"testlabelexclude"}, +}) +@background_task() +def test_label_forwarding_enabled(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + labels = session.get_log_events_labels() + # Excluded label should not appear, and other labels should be prefixed with 'tag.' + assert labels == {"tags.testlabel1": "A", "tags.testlabel2": "B"} + + +@override_application_settings({ + "labels": TEST_LABELS, + "application_logging.forwarding.include_labels.enabled": False, +}) +@background_task() +def test_label_forwarding_disabled(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + labels = session.get_log_events_labels() + # No labels should appear + assert labels == {} From 0b3efd4e5f0b860378427f832315bccbf7f21642 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 17 Oct 2024 09:41:26 -0700 Subject: [PATCH 03/14] Fix unwritten logging validator Expand log event collector validator --- .../agent_features/test_collector_payloads.py | 4 +- tests/testing_support/sample_applications.py | 7 ++- .../validate_log_event_collector_json.py | 52 +++++++++++-------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/tests/agent_features/test_collector_payloads.py b/tests/agent_features/test_collector_payloads.py index 42510e5c7..c6a7ab4dc 100644 --- a/tests/agent_features/test_collector_payloads.py +++ b/tests/agent_features/test_collector_payloads.py @@ -81,8 +81,6 @@ def test_custom_event_json(): custom_event_application.get("/") -@pytest.mark.xfail(reason="Unwritten validator") -@validate_log_event_collector_json +@validate_log_event_collector_json() def test_log_event_json(): normal_application.get("/") - raise NotImplementedError("Fix my validator") diff --git a/tests/testing_support/sample_applications.py b/tests/testing_support/sample_applications.py index 734201e3c..9b3720ef5 100644 --- a/tests/testing_support/sample_applications.py +++ b/tests/testing_support/sample_applications.py @@ -123,8 +123,11 @@ def simple_exceptional_app(environ, start_response): def simple_app_raw(environ, start_response): status = "200 OK" - - _logger.info("Starting response") + + logger = logging.getLogger("simple_app_raw") + logger.setLevel(logging.INFO) + logger.info("Starting response") + start_response(status, response_headers=[]) return [] diff --git a/tests/testing_support/validators/validate_log_event_collector_json.py b/tests/testing_support/validators/validate_log_event_collector_json.py index 00bb3d570..ceda3809b 100644 --- a/tests/testing_support/validators/validate_log_event_collector_json.py +++ b/tests/testing_support/validators/validate_log_event_collector_json.py @@ -15,15 +15,14 @@ import json from newrelic.common.encoding_utils import json_encode -from newrelic.common.object_wrapper import (transient_function_wrapper, - function_wrapper) +from newrelic.common.object_wrapper import transient_function_wrapper, function_wrapper + def validate_log_event_collector_json(num_logs=1): """Validate the format, types and number of logs of the data we send to the collector for harvest. """ - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") def _validate_log_event_collector_json(wrapped, instance, args, kwargs): try: @@ -33,36 +32,45 @@ def _validate_log_event_collector_json(wrapped, instance, args, kwargs): else: samples = list(instance.log_events) - s_info = instance.log_events.sampling_info - agent_run_id = 666 # emulate the payload used in data_collector.py - payload = (agent_run_id, s_info, samples) + payload = ({"logs": tuple(log._asdict() for log in samples)},) collector_json = json_encode(payload) decoded_json = json.loads(collector_json) - assert decoded_json[0] == agent_run_id - - sampling_info = decoded_json[1] - - reservoir_size = instance.settings.application_logging.max_samples_stored - - assert sampling_info["reservoir_size"] == reservoir_size - assert sampling_info["events_seen"] == num_logs - - log_events = decoded_json[2] + log_events = decoded_json[0]["logs"] assert len(log_events) == num_logs for event in log_events: - # event is an array containing intrinsics, user-attributes, - # and agent-attributes - - assert len(event) == 3 - for d in event: - assert isinstance(d, dict) + # event is an array containing timestamp, level, message, attributes + + { + "timestamp": 1729186430797, + "level": "INFO", + "message": "Starting response", + "attributes": { + "entity.type": "SERVICE", + "entity.name": "Python Agent Test (agent_features)", + "entity.guid": "DEVELOPERMODEENTITYGUID", + "hostname": "T23WHWQH20", + "span.id": "51628b750f177405", + "trace.id": "a3b0eb0bd17c433ac461aec42a316069", + }, + } + + assert len(event) == 4 + assert isinstance(event["timestamp"], int) + assert isinstance(event["level"], str) + assert isinstance(event["message"], str) + assert isinstance(event["attributes"], dict) + + expected_attribute_keys = sorted( + ["entity.guid", "entity.name", "entity.type", "hostname", "span.id", "trace.id"] + ) + assert sorted(event["attributes"].keys()) == expected_attribute_keys return result From 7086a10cff8cc364461da8d1ae0aa2577ab4f26b Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 22 Oct 2024 10:59:29 -0700 Subject: [PATCH 04/14] Rename mapping function for general use --- newrelic/config.py | 4 ++-- tests/cross_agent/test_labels_and_rollups.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 12884dac0..30d3c9af6 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -164,7 +164,7 @@ def _map_feature_flag(s): return set(s.split()) -def _map_labels(s): +def _map_as_mapping(s): return newrelic.core.config._environ_as_mapping(name="", default=s) @@ -315,7 +315,7 @@ def _process_setting(section, option, getter, mapper): def _process_configuration(section): _process_setting(section, "feature_flag", "get", _map_feature_flag) _process_setting(section, "app_name", "get", None) - _process_setting(section, "labels", "get", _map_labels) + _process_setting(section, "labels", "get", _map_as_mapping) _process_setting(section, "license_key", "get", _map_default_host_value) _process_setting(section, "api_key", "get", None) _process_setting(section, "host", "get", None) diff --git a/tests/cross_agent/test_labels_and_rollups.py b/tests/cross_agent/test_labels_and_rollups.py index 15ebb1e36..32ca7386b 100644 --- a/tests/cross_agent/test_labels_and_rollups.py +++ b/tests/cross_agent/test_labels_and_rollups.py @@ -16,7 +16,7 @@ import os import pytest -from newrelic.config import _process_labels_setting, _map_labels +from newrelic.config import _process_labels_setting, _map_as_mapping from newrelic.core.config import global_settings from testing_support.fixtures import override_application_settings @@ -41,7 +41,7 @@ def _parametrize_test(test): @pytest.mark.parametrize('name,labelString,warning,expected', _labels_tests) def test_labels(name, labelString, warning, expected): - parsed_labels = _map_labels(labelString) + parsed_labels = _map_as_mapping(labelString) _process_labels_setting(parsed_labels) settings = global_settings() From 4d767221e5e25918eb9e36c8a3b05534ff36937d Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 22 Oct 2024 10:59:49 -0700 Subject: [PATCH 05/14] Add custom attribute setting --- newrelic/config.py | 1 + newrelic/core/config.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index 30d3c9af6..6fd3bbb34 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -546,6 +546,7 @@ def _process_configuration(section): _process_setting(section, "application_logging.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.max_samples_stored", "getint", None) _process_setting(section, "application_logging.forwarding.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.custom_attributes", "get", _map_as_mapping) _process_setting(section, "application_logging.forwarding.include_labels.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.include_labels.exclude", "get", _map_case_insensitive_excl_labels) _process_setting(section, "application_logging.forwarding.context_data.enabled", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 66857ce98..3fe1bc36b 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -940,6 +940,9 @@ def default_otlp_host(host): _settings.application_logging.forwarding.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=True ) +_settings.application_logging.forwarding.custom_attributes = _environ_as_mapping( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES", default="" +) _settings.application_logging.forwarding.include_labels.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_INCLUDE_LABELS_ENABLED", default=False From 19932205bdcfbc151b432a02fe4697239f6750d8 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 22 Oct 2024 11:25:27 -0700 Subject: [PATCH 06/14] Implement global custom attributes for log events --- newrelic/core/data_collector.py | 61 +++++++++++++++++++------ tests/agent_features/test_log_events.py | 55 +++++++++++++++++++--- 2 files changed, 97 insertions(+), 19 deletions(-) diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 2f81fa529..1396f690b 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -32,6 +32,8 @@ from newrelic.core.config import global_settings from newrelic.core.otlp_utils import encode_metric_data, encode_ml_event_data +from newrelic.core.attribute import process_user_attribute, MAX_NUM_USER_ATTRIBUTES + _logger = logging.getLogger(__name__) @@ -154,28 +156,61 @@ def send_dimensional_metric_data(self, start_time, end_time, metric_data): payload = encode_metric_data(metric_data, start_time, end_time) return self._otlp_protocol.send("dimensional_metric_data", payload, path="/v1/metrics") - def get_log_events_labels(self): + def get_log_events_common_block(self): """ "Generate common block for log events.""" - if not self.configuration.application_logging.forwarding.include_labels.enabled: + common = {} + + try: + # Add global custom log attributes to common block + if self.configuration.application_logging.forwarding.custom_attributes: + if self.configuration.high_security: + _logger.debug("Cannot add custom attribute in High Security Mode.") + else: + # Retrieve and process attrs + custom_attributes = {} + for attr_name, attr_value in self.configuration.application_logging.forwarding.custom_attributes: + if len(custom_attributes) >= MAX_NUM_USER_ATTRIBUTES: + _logger.debug("Maximum number of custom attributes already added. Dropping attribute: %r=%r", attr_name, value) + break + + key, val = process_user_attribute(attr_name, attr_value) + + if key is not None: + custom_attributes[key] = val + + common.update(custom_attributes) + + # Add application labels as tags. prefixed attributes to common block + labels = self.configuration.labels + if not labels or not self.configuration.application_logging.forwarding.include_labels.enabled: + return common + elif not self.configuration.application_logging.forwarding.include_labels.exclude: + common.update({ + f"tags.{label['label_type']}": label['label_value'] + for label in labels + }) + else: + common.update({ + f"tags.{label['label_type']}": label['label_value'] + for label in labels + if label['label_type'].lower() not in self.configuration.application_logging.forwarding.include_labels.exclude + }) + + except Exception: + _logger.exception("Cannot generate common block for log events.") return {} - elif not self.configuration.application_logging.forwarding.include_labels.exclude: - return self.configuration.labels or {} else: - return { - f"tags.{label['label_type']}": label['label_value'] - for label in self.configuration.labels - if label['label_type'].lower() not in self.configuration.application_logging.forwarding.include_labels.exclude - } + return common def send_log_events(self, sampling_info, log_event_data): """Called to submit sample set for log events.""" payload = ({"logs": tuple(log._asdict() for log in log_event_data)},) - # Add labels into common block if enabled and not empty - labels = self.get_log_events_labels() - if labels: - payload[0]["common"] = {"attributes": labels} + # Add common block attributes if not empty + common = self.get_log_events_common_block() + if common: + payload[0]["common"] = {"attributes": common} return self._protocol.send("log_event_data", payload) diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index 43a13a80b..79739e5a4 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -153,7 +153,9 @@ def exercise_record_log_event(): ] +# ================================================ # Test Log Forwarding +# ================================================ @enable_log_forwarding @@ -193,7 +195,10 @@ def test(): test() +# ================================================ # Test Message Truncation +# ================================================ + _test_log_event_truncation_events = [{"message": "A" * 32768}] @@ -220,7 +225,9 @@ def test(): test() +# ================================================ # Test Log Forwarding Settings +# ================================================ @disable_log_forwarding @@ -243,7 +250,9 @@ def test(): test() +# ================================================ # Test Log Attribute Settings +# ================================================ @disable_log_attributes @@ -398,7 +407,9 @@ def test(): test() -# Test Log Event Label Settings +# ================================================ +# Test Log Event Labels Settings +# ================================================ # Add labels setting value in already processed format @@ -408,16 +419,30 @@ def test(): @override_application_settings({ "labels": TEST_LABELS, "application_logging.forwarding.include_labels.enabled": True, - "application_logging.forwarding.include_labels.exclude": {"testlabelexclude"}, }) @background_task() def test_label_forwarding_enabled(): txn = current_transaction() session = list(txn.application._agent._applications.values())[0]._active_session - labels = session.get_log_events_labels() + common = session.get_log_events_common_block() + # Excluded label should not appear, and other labels should be prefixed with 'tag.' + assert common == {"tags.testlabel1": "A", "tags.testlabel2": "B", "tags.testlabelexclude": "C"} + + +@override_application_settings({ + "labels": TEST_LABELS, + "application_logging.forwarding.include_labels.enabled": True, + "application_logging.forwarding.include_labels.exclude": {"testlabelexclude"}, +}) +@background_task() +def test_label_forwarding_enabled_exclude(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + common = session.get_log_events_common_block() # Excluded label should not appear, and other labels should be prefixed with 'tag.' - assert labels == {"tags.testlabel1": "A", "tags.testlabel2": "B"} + assert common == {"tags.testlabel1": "A", "tags.testlabel2": "B"} @override_application_settings({ @@ -429,6 +454,24 @@ def test_label_forwarding_disabled(): txn = current_transaction() session = list(txn.application._agent._applications.values())[0]._active_session - labels = session.get_log_events_labels() + common = session.get_log_events_common_block() # No labels should appear - assert labels == {} + assert common == {} + + +# ================================================ +# Test Log Event Global Custom Attributes Settings +# ================================================ + + +@override_application_settings({ + "application_logging.forwarding.custom_attributes": [("custom_attr_1", "value"), ("custom_attr_2", "a" * 256)], +}) +@background_task() +def test_global_custom_attribute_forwarding(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + common = session.get_log_events_common_block() + # Both attrs should appear, and the 2nd attr should be truncated to the max user attribute length + assert common == {"custom_attr_1": "value", "custom_attr_2": "a" * 255} From 0686d186d08a6204da82cf240d9329006fda40ca Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 22 Oct 2024 11:30:57 -0700 Subject: [PATCH 07/14] Remove include_ prefix from logging setting name --- newrelic/config.py | 4 ++-- newrelic/core/config.py | 12 ++++++------ newrelic/core/data_collector.py | 6 +++--- tests/agent_features/test_log_events.py | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 6fd3bbb34..446822bdd 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -547,8 +547,8 @@ def _process_configuration(section): _process_setting(section, "application_logging.forwarding.max_samples_stored", "getint", None) _process_setting(section, "application_logging.forwarding.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.custom_attributes", "get", _map_as_mapping) - _process_setting(section, "application_logging.forwarding.include_labels.enabled", "getboolean", None) - _process_setting(section, "application_logging.forwarding.include_labels.exclude", "get", _map_case_insensitive_excl_labels) + _process_setting(section, "application_logging.forwarding.labels.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.labels.exclude", "get", _map_case_insensitive_excl_labels) _process_setting(section, "application_logging.forwarding.context_data.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.context_data.include", "get", _map_inc_excl_attributes) _process_setting(section, "application_logging.forwarding.context_data.exclude", "get", _map_inc_excl_attributes) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3fe1bc36b..275ec5049 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -323,7 +323,7 @@ class ApplicationLoggingForwardingSettings(Settings): pass -class ApplicationLoggingForwardingIncludeLabelsSettings(Settings): +class ApplicationLoggingForwardingLabelsSettings(Settings): pass @@ -428,7 +428,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.agent_limits = AgentLimitsSettings() _settings.application_logging = ApplicationLoggingSettings() _settings.application_logging.forwarding = ApplicationLoggingForwardingSettings() -_settings.application_logging.forwarding.include_labels = ApplicationLoggingForwardingIncludeLabelsSettings() +_settings.application_logging.forwarding.labels = ApplicationLoggingForwardingLabelsSettings() _settings.application_logging.forwarding.context_data = ApplicationLoggingForwardingContextDataSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() @@ -944,11 +944,11 @@ def default_otlp_host(host): "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES", default="" ) -_settings.application_logging.forwarding.include_labels.enabled = _environ_as_bool( - "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_INCLUDE_LABELS_ENABLED", default=False +_settings.application_logging.forwarding.labels.enabled = _environ_as_bool( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_ENABLED", default=False ) -_settings.application_logging.forwarding.include_labels.exclude = set( - v.lower() for v in _environ_as_set("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_INCLUDE_LABELS_EXCLUDE", default="") +_settings.application_logging.forwarding.labels.exclude = set( + v.lower() for v in _environ_as_set("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_EXCLUDE", default="") ) _settings.application_logging.forwarding.context_data.enabled = _environ_as_bool( diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 1396f690b..6377fc7a7 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -182,9 +182,9 @@ def get_log_events_common_block(self): # Add application labels as tags. prefixed attributes to common block labels = self.configuration.labels - if not labels or not self.configuration.application_logging.forwarding.include_labels.enabled: + if not labels or not self.configuration.application_logging.forwarding.labels.enabled: return common - elif not self.configuration.application_logging.forwarding.include_labels.exclude: + elif not self.configuration.application_logging.forwarding.labels.exclude: common.update({ f"tags.{label['label_type']}": label['label_value'] for label in labels @@ -193,7 +193,7 @@ def get_log_events_common_block(self): common.update({ f"tags.{label['label_type']}": label['label_value'] for label in labels - if label['label_type'].lower() not in self.configuration.application_logging.forwarding.include_labels.exclude + if label['label_type'].lower() not in self.configuration.application_logging.forwarding.labels.exclude }) except Exception: diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index 79739e5a4..d005484db 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -418,7 +418,7 @@ def test(): @override_application_settings({ "labels": TEST_LABELS, - "application_logging.forwarding.include_labels.enabled": True, + "application_logging.forwarding.labels.enabled": True, }) @background_task() def test_label_forwarding_enabled(): @@ -432,8 +432,8 @@ def test_label_forwarding_enabled(): @override_application_settings({ "labels": TEST_LABELS, - "application_logging.forwarding.include_labels.enabled": True, - "application_logging.forwarding.include_labels.exclude": {"testlabelexclude"}, + "application_logging.forwarding.labels.enabled": True, + "application_logging.forwarding.labels.exclude": {"testlabelexclude"}, }) @background_task() def test_label_forwarding_enabled_exclude(): @@ -447,7 +447,7 @@ def test_label_forwarding_enabled_exclude(): @override_application_settings({ "labels": TEST_LABELS, - "application_logging.forwarding.include_labels.enabled": False, + "application_logging.forwarding.labels.enabled": False, }) @background_task() def test_label_forwarding_disabled(): From d632f19d0f2b92d2ae0fdfa4b782e08a0672ab56 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 22 Oct 2024 11:45:38 -0700 Subject: [PATCH 08/14] Add high security mode test for global custom logging attrs --- tests/agent_features/test_log_events.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index d005484db..8cb121710 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -475,3 +475,17 @@ def test_global_custom_attribute_forwarding(): common = session.get_log_events_common_block() # Both attrs should appear, and the 2nd attr should be truncated to the max user attribute length assert common == {"custom_attr_1": "value", "custom_attr_2": "a" * 255} + + +@override_application_settings({ + "high_security": True, + "application_logging.forwarding.custom_attributes": [("custom_attr_1", "value"), ("custom_attr_2", "a" * 256)], +}) +@background_task() +def test_global_custom_attribute_forwarding_high_security_enabled(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + common = session.get_log_events_common_block() + # No custom attrs should be attached with high security enabled + assert common == {} From a460cda55745d7fe82c443ac097c27b172ab75a1 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 24 Oct 2024 14:26:22 -0700 Subject: [PATCH 09/14] Remove unused code --- .../validate_log_event_collector_json.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/testing_support/validators/validate_log_event_collector_json.py b/tests/testing_support/validators/validate_log_event_collector_json.py index ceda3809b..fdc3e4c42 100644 --- a/tests/testing_support/validators/validate_log_event_collector_json.py +++ b/tests/testing_support/validators/validate_log_event_collector_json.py @@ -46,21 +46,6 @@ def _validate_log_event_collector_json(wrapped, instance, args, kwargs): for event in log_events: # event is an array containing timestamp, level, message, attributes - - { - "timestamp": 1729186430797, - "level": "INFO", - "message": "Starting response", - "attributes": { - "entity.type": "SERVICE", - "entity.name": "Python Agent Test (agent_features)", - "entity.guid": "DEVELOPERMODEENTITYGUID", - "hostname": "T23WHWQH20", - "span.id": "51628b750f177405", - "trace.id": "a3b0eb0bd17c433ac461aec42a316069", - }, - } - assert len(event) == 4 assert isinstance(event["timestamp"], int) assert isinstance(event["level"], str) From 4dce5c4553097cefc1e3ce4120fef7b66c60b19b Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 4 Nov 2024 10:31:58 -0800 Subject: [PATCH 10/14] Remove high security checks --- newrelic/core/data_collector.py | 29 +++++++++++-------------- tests/agent_features/test_log_events.py | 14 ------------ 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 6377fc7a7..ecc117f8a 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -163,22 +163,19 @@ def get_log_events_common_block(self): try: # Add global custom log attributes to common block if self.configuration.application_logging.forwarding.custom_attributes: - if self.configuration.high_security: - _logger.debug("Cannot add custom attribute in High Security Mode.") - else: - # Retrieve and process attrs - custom_attributes = {} - for attr_name, attr_value in self.configuration.application_logging.forwarding.custom_attributes: - if len(custom_attributes) >= MAX_NUM_USER_ATTRIBUTES: - _logger.debug("Maximum number of custom attributes already added. Dropping attribute: %r=%r", attr_name, value) - break - - key, val = process_user_attribute(attr_name, attr_value) - - if key is not None: - custom_attributes[key] = val - - common.update(custom_attributes) + # Retrieve and process attrs + custom_attributes = {} + for attr_name, attr_value in self.configuration.application_logging.forwarding.custom_attributes: + if len(custom_attributes) >= MAX_NUM_USER_ATTRIBUTES: + _logger.debug("Maximum number of custom attributes already added. Dropping attribute: %r=%r", attr_name, value) + break + + key, val = process_user_attribute(attr_name, attr_value) + + if key is not None: + custom_attributes[key] = val + + common.update(custom_attributes) # Add application labels as tags. prefixed attributes to common block labels = self.configuration.labels diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index 8cb121710..d005484db 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -475,17 +475,3 @@ def test_global_custom_attribute_forwarding(): common = session.get_log_events_common_block() # Both attrs should appear, and the 2nd attr should be truncated to the max user attribute length assert common == {"custom_attr_1": "value", "custom_attr_2": "a" * 255} - - -@override_application_settings({ - "high_security": True, - "application_logging.forwarding.custom_attributes": [("custom_attr_1", "value"), ("custom_attr_2", "a" * 256)], -}) -@background_task() -def test_global_custom_attribute_forwarding_high_security_enabled(): - txn = current_transaction() - session = list(txn.application._agent._applications.values())[0]._active_session - - common = session.get_log_events_common_block() - # No custom attrs should be attached with high security enabled - assert common == {} From 5a4e2a3c6f80d111dfc43e30ad02f11b3b8b61b7 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Wed, 6 Nov 2024 10:19:29 -0800 Subject: [PATCH 11/14] Add supportability metric for logging labels --- newrelic/core/application.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 4a5632f80..b0961cdbc 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -542,7 +542,9 @@ def connect_to_data_collector(self, activate_agent): application_logging_local_decorating = ( configuration.application_logging.enabled and configuration.application_logging.local_decorating.enabled ) - ai_monitoring_streaming = configuration.ai_monitoring.streaming.enabled + application_logging_labels = ( + application_logging_forwarding and configuration.application_logging.forwarding.labels.enabled + ) internal_metric( f"Supportability/Logging/Forwarding/Python/{'enabled' if application_logging_forwarding else 'disabled'}", 1, @@ -555,6 +557,13 @@ def connect_to_data_collector(self, activate_agent): f"Supportability/Logging/Metrics/Python/{'enabled' if application_logging_metrics else 'disabled'}", 1, ) + internal_metric( + f"Supportability/Logging/Labels/Python/{'enabled' if application_logging_labels else 'disabled'}", + 1, + ) + + # AI monitoring feature toggle metrics + ai_monitoring_streaming = configuration.ai_monitoring.streaming.enabled if not ai_monitoring_streaming: internal_metric( "Supportability/Python/ML/Streaming/Disabled", From fc0d7841e47d6578174ba0b6597cf5e41d6fd831 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:29:34 -0800 Subject: [PATCH 12/14] Update tests/agent_features/test_log_events.py Co-authored-by: Hannah Stepanek --- tests/agent_features/test_log_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index d005484db..4481116d0 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -441,7 +441,7 @@ def test_label_forwarding_enabled_exclude(): session = list(txn.application._agent._applications.values())[0]._active_session common = session.get_log_events_common_block() - # Excluded label should not appear, and other labels should be prefixed with 'tag.' + # Excluded label should not appear, and other labels should be prefixed with 'tags.' assert common == {"tags.testlabel1": "A", "tags.testlabel2": "B"} From b7df4aeb78ef326a969d37eca8b19bd06229f373 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 18 Nov 2024 14:45:28 -0800 Subject: [PATCH 13/14] Expand tests for global custom attributes --- tests/agent_features/test_log_events.py | 34 ++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index 4481116d0..6965df3b9 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -465,13 +465,39 @@ def test_label_forwarding_disabled(): @override_application_settings({ - "application_logging.forwarding.custom_attributes": [("custom_attr_1", "value"), ("custom_attr_2", "a" * 256)], + "application_logging.forwarding.custom_attributes": [("custom_attr_1", "value 1"), ("custom_attr_2", "value 2")], }) @background_task() -def test_global_custom_attribute_forwarding(): +def test_global_custom_attribute_forwarding_enabled(): txn = current_transaction() session = list(txn.application._agent._applications.values())[0]._active_session common = session.get_log_events_common_block() - # Both attrs should appear, and the 2nd attr should be truncated to the max user attribute length - assert common == {"custom_attr_1": "value", "custom_attr_2": "a" * 255} + # Both attrs should appear + assert common == {"custom_attr_1": "value 1", "custom_attr_2": "value 2"} + + +@override_application_settings({ + "application_logging.forwarding.custom_attributes": [("custom_attr_1", "a" * 256)], +}) +@background_task() +def test_global_custom_attribute_forwarding_truncation(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + common = session.get_log_events_common_block() + # Attribute value should be truncated to the max user attribute length + assert common == {"custom_attr_1": "a" * 255} + + +@override_application_settings({ + "application_logging.forwarding.custom_attributes": [(f"custom_attr_{i+1}", "value") for i in range(129)], +}) +@background_task() +def test_global_custom_attribute_forwarding_max_num_attrs(): + txn = current_transaction() + session = list(txn.application._agent._applications.values())[0]._active_session + + common = session.get_log_events_common_block() + # Should be truncated to the max number of user attributes + assert common == {f"custom_attr_{i+1}": "value" for i in range(128)} From a4b352b4e8e26ab8239419f252b521417e3996aa Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 18 Nov 2024 14:45:46 -0800 Subject: [PATCH 14/14] Fix exception in logger message --- newrelic/core/data_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 69c4376d0..32788ca2e 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -167,7 +167,7 @@ def get_log_events_common_block(self): custom_attributes = {} for attr_name, attr_value in self.configuration.application_logging.forwarding.custom_attributes: if len(custom_attributes) >= MAX_NUM_USER_ATTRIBUTES: - _logger.debug("Maximum number of custom attributes already added. Dropping attribute: %r=%r", attr_name, value) + _logger.debug("Maximum number of custom attributes already added. Dropping attribute: %r=%r", attr_name, attr_value) break key, val = process_user_attribute(attr_name, attr_value)