From b709b00de6d2707807631a1edc286ff6ec7506a4 Mon Sep 17 00:00:00 2001 From: Allan Feldman <6374032+a-feld@users.noreply.github.com> Date: Tue, 19 Jan 2021 18:25:39 -0500 Subject: [PATCH] Improve startup performance by deferring load of urllib3. (#124) * Improve startup performance by deferring load of urllib3 and friends. Loading urllib3 can be expensive since there are regex compiles at import time. urllib3 is not immediately needed for operations at initialize, therefore this commit delays the loading of urllib3 and later only loads those imports on a background thread. * Fix tests relying on multiprocessing import. --- newrelic/common/constants.py | 9 ++++ newrelic/common/system_info.py | 49 +------------------ newrelic/common/utilization.py | 36 ++++++++++++++ newrelic/core/agent_protocol.py | 3 +- newrelic/core/application.py | 7 ++- newrelic/core/database_node.py | 4 +- newrelic/core/datastore_node.py | 4 +- tests/agent_unittests/test_environment.py | 3 ++ .../test_boot_id_utilization_data.py | 2 +- tests/cross_agent/test_utilization_configs.py | 3 +- tests/testing_support/fixtures.py | 2 +- .../validate_slow_sql_collector_json.py | 2 +- 12 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 newrelic/common/constants.py diff --git a/newrelic/common/constants.py b/newrelic/common/constants.py new file mode 100644 index 0000000000..cf006fc104 --- /dev/null +++ b/newrelic/common/constants.py @@ -0,0 +1,9 @@ +LOCALHOST_EQUIVALENTS = frozenset(( + 'localhost', + '127.0.0.1', + '0.0.0.0', + '0:0:0:0:0:0:0:0', + '0:0:0:0:0:0:0:1', + '::1', + '::', +)) diff --git a/newrelic/common/system_info.py b/newrelic/common/system_info.py index 3fb8419cd8..9b33514be6 100644 --- a/newrelic/common/system_info.py +++ b/newrelic/common/system_info.py @@ -18,7 +18,6 @@ """ import logging -import multiprocessing import os import re import socket @@ -26,8 +25,6 @@ import sys import threading -from newrelic.common.utilization import CommonUtilization - try: from subprocess import check_output as _execute_program except ImportError: @@ -56,21 +53,12 @@ def _execute_program(*popenargs, **kwargs): _logger = logging.getLogger(__name__) -LOCALHOST_EQUIVALENTS = set([ - 'localhost', - '127.0.0.1', - '0.0.0.0', - '0:0:0:0:0:0:0:0', - '0:0:0:0:0:0:0:1', - '::1', - '::', -]) - def logical_processor_count(): """Returns the number of logical processors in the system. """ + import multiprocessing # The multiprocessing module provides support for Windows, # BSD systems (including MacOS X) and systems which support @@ -423,38 +411,3 @@ def getips(): s.close() return _nr_cached_ip_address - - -class BootIdUtilization(CommonUtilization): - VENDOR_NAME = 'boot_id' - METADATA_URL = '/proc/sys/kernel/random/boot_id' - - @classmethod - def fetch(cls): - if not sys.platform.startswith('linux'): - return - - try: - with open(cls.METADATA_URL, 'rb') as f: - return f.readline().decode('ascii') - except: - # There are all sorts of exceptions that can occur here - # (i.e. permissions, non-existent file, etc) - cls.record_error(cls.METADATA_URL, 'File read error.') - pass - - @staticmethod - def get_values(value): - return value - - @classmethod - def sanitize(cls, value): - if value is None: - return - - stripped = value.strip() - - if len(stripped) != 36: - cls.record_error(cls.METADATA_URL, stripped) - - return stripped[:128] or None diff --git a/newrelic/common/utilization.py b/newrelic/common/utilization.py index adba4443c9..f60fc6c2d1 100644 --- a/newrelic/common/utilization.py +++ b/newrelic/common/utilization.py @@ -17,6 +17,7 @@ import re import socket import string +import sys import threading from newrelic.common.agent_http import InsecureHttpClient @@ -301,3 +302,38 @@ def get_values(cls, v): v = v.decode('utf-8') return {'kubernetes_service_host': v} + + +class BootIdUtilization(CommonUtilization): + VENDOR_NAME = 'boot_id' + METADATA_URL = '/proc/sys/kernel/random/boot_id' + + @classmethod + def fetch(cls): + if not sys.platform.startswith('linux'): + return + + try: + with open(cls.METADATA_URL, 'rb') as f: + return f.readline().decode('ascii') + except: + # There are all sorts of exceptions that can occur here + # (i.e. permissions, non-existent file, etc) + cls.record_error(cls.METADATA_URL, 'File read error.') + pass + + @staticmethod + def get_values(value): + return value + + @classmethod + def sanitize(cls, value): + if value is None: + return + + stripped = value.strip() + + if len(stripped) != 36: + cls.record_error(cls.METADATA_URL, stripped) + + return stripped[:128] or None diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index b72aee2cc1..64be7a0173 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -27,6 +27,7 @@ from newrelic.common.utilization import ( AWSUtilization, AzureUtilization, + BootIdUtilization, DockerUtilization, GCPUtilization, KubernetesUtilization, @@ -302,7 +303,7 @@ def _connect_payload(app_name, linked_applications, environment, settings): if ip_address: utilization_settings["ip_address"] = ip_address - boot_id = system_info.BootIdUtilization.detect() + boot_id = BootIdUtilization.detect() if boot_id: utilization_settings["boot_id"] = boot_id diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 07dda2a11a..f661f95276 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -31,7 +31,6 @@ from newrelic.core.config import global_settings from newrelic.core.custom_event import create_custom_event -from newrelic.core.data_collector import create_session from newrelic.network.exceptions import (ForceAgentRestart, ForceAgentDisconnect, DiscardDataForRequest, RetryDataForRequest, NetworkInterfaceException) @@ -272,6 +271,12 @@ def connect_to_data_collector(self, activate_agent): """ + # Defer these imports so that they are loaded on the connect thread + # instead of the main thread + # + # This speeds up startup since some of these imports are slow + from newrelic.core.data_collector import create_session + if self._agent_shutdown: return diff --git a/newrelic/core/database_node.py b/newrelic/core/database_node.py index b0f2d4aa40..65253fa774 100644 --- a/newrelic/core/database_node.py +++ b/newrelic/core/database_node.py @@ -17,7 +17,7 @@ import newrelic.core.attribute as attribute import newrelic.core.trace_node -from newrelic.common import system_info +from newrelic.common import constants, system_info from newrelic.core.database_utils import sql_statement, explain_plan from newrelic.core.node_mixin import DatastoreNodeMixin from newrelic.core.metric import TimeMetric @@ -67,7 +67,7 @@ def product(self): @property def instance_hostname(self): - if self.host in system_info.LOCALHOST_EQUIVALENTS: + if self.host in constants.LOCALHOST_EQUIVALENTS: hostname = system_info.gethostname() else: hostname = self.host diff --git a/newrelic/core/datastore_node.py b/newrelic/core/datastore_node.py index 9135cc1e4d..c2d99cba41 100644 --- a/newrelic/core/datastore_node.py +++ b/newrelic/core/datastore_node.py @@ -16,7 +16,7 @@ import newrelic.core.trace_node -from newrelic.common import system_info +from newrelic.common import constants, system_info from newrelic.core.node_mixin import DatastoreNodeMixin from newrelic.core.metric import TimeMetric @@ -30,7 +30,7 @@ class DatastoreNode(_DatastoreNode, DatastoreNodeMixin): @property def instance_hostname(self): - if self.host in system_info.LOCALHOST_EQUIVALENTS: + if self.host in constants.LOCALHOST_EQUIVALENTS: hostname = system_info.gethostname() else: hostname = self.host diff --git a/tests/agent_unittests/test_environment.py b/tests/agent_unittests/test_environment.py index ef5c5e4488..fb5a863a67 100644 --- a/tests/agent_unittests/test_environment.py +++ b/tests/agent_unittests/test_environment.py @@ -56,6 +56,9 @@ def get(self, *args, **kwargs): def __getitem__(self, *args, **kwargs): return self.d.__getitem__(*args, **kwargs) + def __setitem__(self, *args, **kwargs): + return self.d.__setitem__(*args, **kwargs) + def __contains__(self, *args, **kwargs): return self.d.__contains__(*args, **kwargs) diff --git a/tests/cross_agent/test_boot_id_utilization_data.py b/tests/cross_agent/test_boot_id_utilization_data.py index 2eaeae6707..2a60478c06 100644 --- a/tests/cross_agent/test_boot_id_utilization_data.py +++ b/tests/cross_agent/test_boot_id_utilization_data.py @@ -18,7 +18,7 @@ import sys import tempfile -from newrelic.common.system_info import BootIdUtilization +from newrelic.common.utilization import BootIdUtilization from testing_support.fixtures import validate_internal_metrics diff --git a/tests/cross_agent/test_utilization_configs.py b/tests/cross_agent/test_utilization_configs.py index 810631ee68..602321084f 100644 --- a/tests/cross_agent/test_utilization_configs.py +++ b/tests/cross_agent/test_utilization_configs.py @@ -23,8 +23,7 @@ # methods in newrelic.core.data_collector and does not put them back! from testing_support.mock_http_client import create_client_cls from newrelic.core.agent_protocol import AgentProtocol -from newrelic.common.system_info import BootIdUtilization -from newrelic.common.utilization import (CommonUtilization) +from newrelic.common.utilization import (BootIdUtilization, CommonUtilization) from newrelic.common.object_wrapper import (function_wrapper) import newrelic.core.config diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 6f3c6062d2..947667c82d 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -41,7 +41,7 @@ from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import (transient_function_wrapper, function_wrapper, wrap_function_wrapper, ObjectProxy) -from newrelic.common.system_info import LOCALHOST_EQUIVALENTS +from newrelic.common.constants import LOCALHOST_EQUIVALENTS from newrelic.config import initialize diff --git a/tests/testing_support/validators/validate_slow_sql_collector_json.py b/tests/testing_support/validators/validate_slow_sql_collector_json.py index 40cef39d2c..f0eef300ca 100644 --- a/tests/testing_support/validators/validate_slow_sql_collector_json.py +++ b/tests/testing_support/validators/validate_slow_sql_collector_json.py @@ -15,7 +15,7 @@ from newrelic.common.encoding_utils import unpack_field from newrelic.common.object_wrapper import transient_function_wrapper -from newrelic.common.system_info import LOCALHOST_EQUIVALENTS +from newrelic.common.constants import LOCALHOST_EQUIVALENTS from newrelic.core.database_utils import SQLConnections from newrelic.packages import six