From cd23f56e6d84133de4a10169970f5b3dc9330a74 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 21 Jul 2022 09:56:09 -0700 Subject: [PATCH] Enable log forwarding by default. (#587) * Enable log forwarding by default. (#583) * Enable log forwarding by default. * Fix catmap fixture for log forwarding compatibility. * Use isinstance check in fixtures. * Update data usage supportability metrics. (#582) * Update data usage supportability metric name. * Fix compression bug and add testing. Co-authored-by: Lalleh Rafeei Co-authored-by: Nyenty Ayuk-Enow * Update harvest_loop tests. * Update compression test to check method1 payload length for consistency. Co-authored-by: Lalleh Rafeei Co-authored-by: Nyenty Ayuk-Enow Co-authored-by: Lalleh Rafeei Co-authored-by: Nyenty Ayuk-Enow --- newrelic/common/agent_http.py | 22 ++++++++---- newrelic/core/config.py | 2 +- tests/agent_unittests/test_harvest_loop.py | 2 +- tests/agent_unittests/test_http_client.py | 39 +++++++++++++++------- tests/testing_support/fixtures.py | 4 +-- 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/newrelic/common/agent_http.py b/newrelic/common/agent_http.py index 2398920007..e9d9a00aac 100644 --- a/newrelic/common/agent_http.py +++ b/newrelic/common/agent_http.py @@ -524,24 +524,32 @@ def _supportability_request(params, payload, body, compression_time): # ********* # Used only for supportability metrics. Do not use to drive business # logic! + # payload: uncompressed + # body: compressed agent_method = params and params.get("method") # ********* - if agent_method and body: + if agent_method and payload: # Compression was applied if compression_time is not None: internal_metric( - "Supportability/Python/Collector/ZLIB/Bytes/%s" % agent_method, - len(payload), + "Supportability/Python/Collector/%s/ZLIB/Bytes" % agent_method, + len(body), ) internal_metric( - "Supportability/Python/Collector/ZLIB/Compress/%s" % agent_method, + "Supportability/Python/Collector/ZLIB/Bytes", len(body) + ) + internal_metric( + "Supportability/Python/Collector/%s/ZLIB/Compress" % agent_method, compression_time, ) - internal_metric( - "Supportability/Python/Collector/Output/Bytes/%s" % agent_method, - len(body), + "Supportability/Python/Collector/%s/Output/Bytes" % agent_method, + len(payload), + ) + # Top level metric to aggregate overall bytes being sent + internal_metric( + "Supportability/Python/Collector/Output/Bytes", len(payload) ) @staticmethod diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 57f408b872..60520c1134 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -812,7 +812,7 @@ def default_host(license_key): _settings.application_logging.enabled = _environ_as_bool("NEW_RELIC_APPLICATION_LOGGING_ENABLED", default=True) _settings.application_logging.forwarding.enabled = _environ_as_bool( - "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=False + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=True ) _settings.application_logging.metrics.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED", default=True diff --git a/tests/agent_unittests/test_harvest_loop.py b/tests/agent_unittests/test_harvest_loop.py index 7a1f1702d0..7760e13078 100644 --- a/tests/agent_unittests/test_harvest_loop.py +++ b/tests/agent_unittests/test_harvest_loop.py @@ -896,7 +896,7 @@ def test_default_events_harvested(allowlist_event): num_seen = 0 if (allowlist_event != "span_event_data") else 1 assert app._stats_engine.span_events.num_seen == num_seen - assert app._stats_engine.metrics_count() == 1 + assert app._stats_engine.metrics_count() == 4 @failing_endpoint("analytic_event_data") diff --git a/tests/agent_unittests/test_http_client.py b/tests/agent_unittests/test_http_client.py index b1fc4b4f4e..a5c340d6a6 100644 --- a/tests/agent_unittests/test_http_client.py +++ b/tests/agent_unittests/test_http_client.py @@ -289,32 +289,47 @@ def test_http_payload_compression(server, client_cls, method, threshold): compression_threshold=threshold, ) as client: with InternalTraceContext(internal_metrics): - status, data = client.send_request(payload=payload, params={"method": "test"}) + status, data = client.send_request(payload=payload, params={"method": "method1"}) + + # Sending one additional request to valid metric aggregation for top level data usage supportability metrics + with client_cls( + "localhost", + server.port, + disable_certificate_validation=True, + compression_method=method, + compression_threshold=threshold, + ) as client: + with InternalTraceContext(internal_metrics): + status, data = client.send_request(payload=payload, params={"method": "method2"}) assert status == 200 data = data.split(b"\n") sent_payload = data[-1] payload_byte_len = len(sent_payload) - internal_metrics = dict(internal_metrics.metrics()) if client_cls is ApplicationModeClient: - assert internal_metrics["Supportability/Python/Collector/Output/Bytes/test"][:2] == [ + assert internal_metrics["Supportability/Python/Collector/method1/Output/Bytes"][:2] == [ 1, - payload_byte_len, + len(payload), + ] + assert internal_metrics["Supportability/Python/Collector/Output/Bytes"][:2] == [ + 2, + len(payload)*2, ] if threshold < 20: # Verify compression time is recorded - assert internal_metrics["Supportability/Python/Collector/ZLIB/Compress/test"][0] == 1 - assert internal_metrics["Supportability/Python/Collector/ZLIB/Compress/test"][1] > 0 - - # Verify the original payload length is recorded - assert internal_metrics["Supportability/Python/Collector/ZLIB/Bytes/test"][:2] == [1, len(payload)] - - assert len(internal_metrics) == 3 + assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Compress"][0] == 1 + assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Compress"][1] > 0 + + # Verify the compressed payload length is recorded + assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Bytes"][:2] == [1, payload_byte_len] + assert internal_metrics["Supportability/Python/Collector/ZLIB/Bytes"][:2] == [2, payload_byte_len*2] + + assert len(internal_metrics) == 8 else: # Verify no ZLIB compression metrics were sent - assert len(internal_metrics) == 1 + assert len(internal_metrics) == 3 else: assert not internal_metrics diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 71bfea6705..a897303774 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -2625,8 +2625,8 @@ def _validate_analytics_sample_data(wrapped, instance, args, kwargs): _new_wrapped = _capture_samples(wrapped) result = _new_wrapped(*args, **kwargs) - - _samples = [s for s in samples if s[0]["type"] == "Transaction"] + # Check type of s[0] because it returns an integer if s is a LogEventNode + _samples = [s for s in samples if not isinstance(s[0], int) and s[0]["type"] == "Transaction"] assert _samples, "No Transaction events captured." for sample in _samples: assert isinstance(sample, list)