Merge branch 'master' into dongothing-dd/add-hosts

.github/workflows/test.yml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-22.04]
         python-version: ['pypy2.7', '3.7', 'pypy3.8']
         # os: [ubuntu-latest, windows-latest, macos-latest]
         # python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', 'pypy-2.7', 'pypy-3.8']

datadog/api/http_client.py

@@ -27,6 +27,11 @@
 except ImportError:
     urlfetch, urlfetch_errors = None, None
 
+try:
+    import urllib3  # type: ignore
+except ImportError:
+    urllib3 = None
+
 # datadog
 from datadog.api.exceptions import ProxyError, ClientError, HTTPError, HttpTimeout
 
@@ -178,6 +183,58 @@ def raise_on_status(cls, result):
                 raise HTTPError(status_code)
 
 
+class Urllib3Client(HTTPClient):
+    """
+    HTTP client based on 3rd party `urllib3` module.
+    """
+
+    _pool = None
+    _pool_lock = Lock()
+
+    @classmethod
+    def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries):
+        """
+        Wrapper around `urllib3.PoolManager.request` method. This method will raise
+        exceptions for HTTP status codes that are not 2xx.
+        """
+        try:
+            with cls._pool_lock:
+                if cls._pool is None:
+                    cls._pool = urllib3.PoolManager(
+                        retries=max_retries,
+                        timeout=timeout,
+                        cert_reqs="CERT_REQUIRED" if verify else "CERT_NONE",
+                    )
+
+            newheaders = copy.deepcopy(headers)
+            newheaders["User-Agent"] = _get_user_agent_header()
+            response = cls._pool.request(
+                method, url, body=data, fields=params, headers=newheaders
+            )
+            cls.raise_on_status(response)
+
+        except urllib3.exceptions.ProxyError as e:
+            raise _remove_context(ProxyError(method, url, e))
+        except urllib3.exceptions.MaxRetryError as e:
+            raise _remove_context(ClientError(method, url, e))
+        except urllib3.exceptions.TimeoutError as e:
+            raise _remove_context(HttpTimeout(method, url, e))
+        except urllib3.exceptions.HTTPError as e:
+            raise _remove_context(HTTPError(e))
+
+        return response
+
+    @classmethod
+    def raise_on_status(cls, response):
+        """
+        Raise on HTTP status code errors.
+        """
+        status_code = response.status
+        if status_code < 200 or status_code >= 300:
+            if status_code not in (400, 401, 403, 404, 409, 429):
+                raise HTTPError(status_code, response.reason)
+
+
 def resolve_http_client():
     """
     Resolve an appropriate HTTP client based the defined priority and user environment.
@@ -190,6 +247,10 @@ def resolve_http_client():
         log.debug(u"Use `urlfetch` based HTTP client.")
         return URLFetchClient
 
+    if urllib3:
+        log.debug(u"Use `urllib3` based HTTP client.")
+        return Urllib3Client
+
     raise ImportError(
         u"Datadog API client was unable to resolve a HTTP client. " u" Please install `requests` library."
     )

datadog/util/compat.py

@@ -5,9 +5,7 @@
 """
 Imports for compatibility with Python 2, Python 3 and Google App Engine.
 """
-from functools import wraps
 import logging
-import socket
 import sys
 
 # Logging
@@ -23,11 +21,25 @@
 if sys.version_info[0] >= 3:
     import builtins
     from collections import UserDict as IterableUserDict
-    import configparser
-    from configparser import ConfigParser
     from io import StringIO
-    from urllib.parse import urljoin, urlparse
-    import urllib.request as url_lib, urllib.error, urllib.parse
+    from urllib.parse import urlparse
+
+    class LazyLoader(object):
+        def __init__(self, module_name):
+            self.module_name = module_name
+
+        def __getattr__(self, name):
+            # defer the importing of the module to when one of its attributes
+            # is accessed
+            import importlib
+            mod = importlib.import_module(self.module_name)
+            return getattr(mod, name)
+
+    url_lib = LazyLoader('urllib.request')
+    configparser = LazyLoader('configparser')
+
+    def ConfigParser():
+        return configparser.ConfigParser()
 
     imap = map
     get_input = input
@@ -48,7 +60,7 @@ def iternext(iter):
     from cStringIO import StringIO
     from itertools import imap
     import urllib2 as url_lib
-    from urlparse import urljoin, urlparse
+    from urlparse import urlparse
     from UserDict import IterableUserDict
 
     get_input = raw_input
@@ -63,7 +75,7 @@ def iternext(iter):
 
 # Python >= 3.5
 if sys.version_info >= (3, 5):
-    from asyncio import iscoroutinefunction
+    from inspect import iscoroutinefunction
 # Others
 else:
 
@@ -76,9 +88,7 @@ def iscoroutinefunction(*args, **kwargs):
     from logging import NullHandler
 # Python 2.6.x
 else:
-    from logging import Handler
-
-    class NullHandler(Handler):
+    class NullHandler(logging.Handler):
         def emit(self, record):
             pass
 

tests/unit/util/test_compat.py

@@ -3,11 +3,13 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2015-Present Datadog, Inc
 import logging
+import pytest
+import sys
 import unittest
 
 from mock import patch
 
-from datadog.util.compat import conditional_lru_cache, is_higher_py32
+from datadog.util.compat import conditional_lru_cache, is_higher_py32, is_p3k
 
 class TestConditionalLRUCache(unittest.TestCase):
     def test_normal_usage(self):
@@ -48,3 +50,36 @@ def test_function():
                 mock_debug.assert_called_once()
             else:
                 mock_debug.assert_not_called()
+
+@pytest.mark.skipif(not is_p3k(), reason='Python 3 only')
+def test_slow_imports(monkeypatch):
+    # We should lazy load certain modules to avoid slowing down the startup
+    # time when running in a serverless environment.  This test will fail if
+    # any of those modules are imported during the import of datadogpy.
+
+    blocklist = [
+        'configparser',
+        'email.mime.application',
+        'email.mime.multipart',
+        'importlib.metadata',
+        'importlib_metadata',
+        'logging.handlers',
+        'multiprocessing',
+        'urllib.request',
+    ]
+
+    class BlockListFinder:
+        def find_spec(self, fullname, *args):
+            for lib in blocklist:
+                if fullname == lib:
+                    raise ImportError('module %s was imported!' % fullname)
+            return None
+        find_module = find_spec  # Python 2
+
+    monkeypatch.setattr('sys.meta_path', [BlockListFinder()] + sys.meta_path)
+
+    for mod in sys.modules.copy():
+        if mod in blocklist or mod.startswith('datadog'):
+            del sys.modules[mod]
+
+    import datadog