Improve startup performance by deferring load of urllib3. (#124)

* Improve startup performance by deferring load of urllib3 and friends.

Loading urllib3 can be expensive since there are regex compiles at
import time.

urllib3 is not immediately needed for operations at initialize,
therefore this commit delays the loading of urllib3 and later only loads
those imports on a background thread.

* Fix tests relying on multiprocessing import.

newrelic/common/constants.py

@@ -0,0 +1,9 @@
+LOCALHOST_EQUIVALENTS = frozenset((
+    'localhost',
+    '127.0.0.1',
+    '0.0.0.0',
+    '0:0:0:0:0:0:0:0',
+    '0:0:0:0:0:0:0:1',
+    '::1',
+    '::',
+))

newrelic/common/system_info.py

@@ -18,16 +18,13 @@
 """
 
 import logging
-import multiprocessing
 import os
 import re
 import socket
 import subprocess
 import sys
 import threading
 
-from newrelic.common.utilization import CommonUtilization
-
 try:
     from subprocess import check_output as _execute_program
 except ImportError:
@@ -56,21 +53,12 @@ def _execute_program(*popenargs, **kwargs):
 
 _logger = logging.getLogger(__name__)
 
-LOCALHOST_EQUIVALENTS = set([
-    'localhost',
-    '127.0.0.1',
-    '0.0.0.0',
-    '0:0:0:0:0:0:0:0',
-    '0:0:0:0:0:0:0:1',
-    '::1',
-    '::',
-])
-
 
 def logical_processor_count():
     """Returns the number of logical processors in the system.
 
     """
+    import multiprocessing
 
     # The multiprocessing module provides support for Windows,
     # BSD systems (including MacOS X) and systems which support
@@ -423,38 +411,3 @@ def getips():
                 s.close()
 
     return _nr_cached_ip_address
-
-
-class BootIdUtilization(CommonUtilization):
-    VENDOR_NAME = 'boot_id'
-    METADATA_URL = '/proc/sys/kernel/random/boot_id'
-
-    @classmethod
-    def fetch(cls):
-        if not sys.platform.startswith('linux'):
-            return
-
-        try:
-            with open(cls.METADATA_URL, 'rb') as f:
-                return f.readline().decode('ascii')
-        except:
-            # There are all sorts of exceptions that can occur here
-            # (i.e. permissions, non-existent file, etc)
-            cls.record_error(cls.METADATA_URL, 'File read error.')
-            pass
-
-    @staticmethod
-    def get_values(value):
-        return value
-
-    @classmethod
-    def sanitize(cls, value):
-        if value is None:
-            return
-
-        stripped = value.strip()
-
-        if len(stripped) != 36:
-            cls.record_error(cls.METADATA_URL, stripped)
-
-        return stripped[:128] or None

newrelic/common/utilization.py

@@ -17,6 +17,7 @@
 import re
 import socket
 import string
+import sys
 import threading
 
 from newrelic.common.agent_http import InsecureHttpClient
@@ -301,3 +302,38 @@ def get_values(cls, v):
             v = v.decode('utf-8')
 
         return {'kubernetes_service_host': v}
+
+
+class BootIdUtilization(CommonUtilization):
+    VENDOR_NAME = 'boot_id'
+    METADATA_URL = '/proc/sys/kernel/random/boot_id'
+
+    @classmethod
+    def fetch(cls):
+        if not sys.platform.startswith('linux'):
+            return
+
+        try:
+            with open(cls.METADATA_URL, 'rb') as f:
+                return f.readline().decode('ascii')
+        except:
+            # There are all sorts of exceptions that can occur here
+            # (i.e. permissions, non-existent file, etc)
+            cls.record_error(cls.METADATA_URL, 'File read error.')
+            pass
+
+    @staticmethod
+    def get_values(value):
+        return value
+
+    @classmethod
+    def sanitize(cls, value):
+        if value is None:
+            return
+
+        stripped = value.strip()
+
+        if len(stripped) != 36:
+            cls.record_error(cls.METADATA_URL, stripped)
+
+        return stripped[:128] or None

newrelic/core/agent_protocol.py

@@ -27,6 +27,7 @@
 from newrelic.common.utilization import (
     AWSUtilization,
     AzureUtilization,
+    BootIdUtilization,
     DockerUtilization,
     GCPUtilization,
     KubernetesUtilization,
@@ -302,7 +303,7 @@ def _connect_payload(app_name, linked_applications, environment, settings):
         if ip_address:
             utilization_settings["ip_address"] = ip_address
 
-        boot_id = system_info.BootIdUtilization.detect()
+        boot_id = BootIdUtilization.detect()
         if boot_id:
             utilization_settings["boot_id"] = boot_id
 

newrelic/core/application.py

@@ -31,7 +31,6 @@
 
 from newrelic.core.config import global_settings
 from newrelic.core.custom_event import create_custom_event
-from newrelic.core.data_collector import create_session
 from newrelic.network.exceptions import (ForceAgentRestart,
         ForceAgentDisconnect, DiscardDataForRequest, RetryDataForRequest,
         NetworkInterfaceException)
@@ -272,6 +271,12 @@ def connect_to_data_collector(self, activate_agent):
 
         """
 
+        # Defer these imports so that they are loaded on the connect thread
+        # instead of the main thread
+        #
+        # This speeds up startup since some of these imports are slow
+        from newrelic.core.data_collector import create_session
+
         if self._agent_shutdown:
             return
 

newrelic/core/database_node.py

@@ -17,7 +17,7 @@
 import newrelic.core.attribute as attribute
 import newrelic.core.trace_node
 
-from newrelic.common import system_info
+from newrelic.common import constants, system_info
 from newrelic.core.database_utils import sql_statement, explain_plan
 from newrelic.core.node_mixin import DatastoreNodeMixin
 from newrelic.core.metric import TimeMetric
@@ -67,7 +67,7 @@ def product(self):
 
     @property
     def instance_hostname(self):
-        if self.host in system_info.LOCALHOST_EQUIVALENTS:
+        if self.host in constants.LOCALHOST_EQUIVALENTS:
             hostname = system_info.gethostname()
         else:
             hostname = self.host

newrelic/core/datastore_node.py

@@ -16,7 +16,7 @@
 
 import newrelic.core.trace_node
 
-from newrelic.common import system_info
+from newrelic.common import constants, system_info
 from newrelic.core.node_mixin import DatastoreNodeMixin
 from newrelic.core.metric import TimeMetric
 
@@ -30,7 +30,7 @@ class DatastoreNode(_DatastoreNode, DatastoreNodeMixin):
 
     @property
     def instance_hostname(self):
-        if self.host in system_info.LOCALHOST_EQUIVALENTS:
+        if self.host in constants.LOCALHOST_EQUIVALENTS:
             hostname = system_info.gethostname()
         else:
             hostname = self.host

tests/agent_unittests/test_environment.py

@@ -56,6 +56,9 @@ def get(self, *args, **kwargs):
     def __getitem__(self, *args, **kwargs):
         return self.d.__getitem__(*args, **kwargs)
 
+    def __setitem__(self, *args, **kwargs):
+        return self.d.__setitem__(*args, **kwargs)
+
     def __contains__(self, *args, **kwargs):
         return self.d.__contains__(*args, **kwargs)
 

tests/cross_agent/test_boot_id_utilization_data.py

@@ -18,7 +18,7 @@
 import sys
 import tempfile
 
-from newrelic.common.system_info import BootIdUtilization
+from newrelic.common.utilization import BootIdUtilization
 
 from testing_support.fixtures import validate_internal_metrics
 

tests/cross_agent/test_utilization_configs.py

@@ -23,8 +23,7 @@
 # methods in newrelic.core.data_collector and does not put them back!
 from testing_support.mock_http_client import create_client_cls
 from newrelic.core.agent_protocol import AgentProtocol
-from newrelic.common.system_info import BootIdUtilization
-from newrelic.common.utilization import (CommonUtilization)
+from newrelic.common.utilization import (BootIdUtilization, CommonUtilization)
 from newrelic.common.object_wrapper import (function_wrapper)
 import newrelic.core.config
 

tests/testing_support/fixtures.py

@@ -41,7 +41,7 @@
 from newrelic.common.object_names import callable_name
 from newrelic.common.object_wrapper import (transient_function_wrapper,
         function_wrapper, wrap_function_wrapper, ObjectProxy)
-from newrelic.common.system_info import LOCALHOST_EQUIVALENTS
+from newrelic.common.constants import LOCALHOST_EQUIVALENTS
 
 from newrelic.config import initialize
 

tests/testing_support/validators/validate_slow_sql_collector_json.py

@@ -15,7 +15,7 @@
 
 from newrelic.common.encoding_utils import unpack_field
 from newrelic.common.object_wrapper import transient_function_wrapper
-from newrelic.common.system_info import LOCALHOST_EQUIVALENTS
+from newrelic.common.constants import LOCALHOST_EQUIVALENTS
 from newrelic.core.database_utils import SQLConnections
 from newrelic.packages import six