From 37f0173ecd7964bb046f7cee4e65c29324c176b1 Mon Sep 17 00:00:00 2001 From: Ryan Hall Date: Mon, 18 Nov 2024 15:43:29 -0500 Subject: [PATCH] AMLII-2173 - Add DD_DOGSTATSD_URL support to datadogpy Adds support for the unix:// and udp:// variants of DD_DOGSTATSD_URL. Will only be applied if the host and port are their default values and no socket_path has been provided. --- README.md | 11 +++ datadog/dogstatsd/base.py | 101 +++++++++++++++++++++++----- doc/source/index.rst | 9 +++ tests/unit/dogstatsd/test_statsd.py | 52 +++++++++++++- 4 files changed, 154 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2bb40165b..a123cdded 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ options = { initialize(**options) ``` +Alternatively, the environment variable `DD_DOGSTATSD_URL` can be used to define a udp connection: +`DD_DOGSTATSD_URL=udp://localhost:8125` + +Manually supplying a host/port will take precedence over using this environment variable. + See the full list of available [DogStatsD client instantiation parameters](https://docs.datadoghq.com/developers/dogstatsd/?code-lang=python#client-instantiation-parameters). #### Instantiate the DogStatsd client with UDS @@ -110,6 +115,12 @@ options = { initialize(**options) ``` +Alternatively, the environment variable `DD_DOGSTATSD_URL` can be used to define a UDS connection: +`DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket` + +As with the udp variant, manually supplying a statsd_socket_path will take precedence over the environment variable. + + #### Origin detection over UDP and UDS Origin detection is a method to detect which pod `DogStatsD` packets are coming from in order to add the pod's tags to the tag list. diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 44e07f7cf..41bbce4aa 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -25,6 +25,11 @@ # pypy has the same module, but capitalized. import Queue as queue # type: ignore[no-redef] +try: + from urllib.parse import urlparse # type: ignore +except ImportError: + # Python 2 has the same functionality stored under a different module. + from urlparse import urlparse # type: ignore # pylint: disable=unused-import from typing import Optional, List, Text, Union @@ -54,6 +59,7 @@ UNIX_ADDRESS_SCHEME = "unix://" UNIX_ADDRESS_DATAGRAM_SCHEME = "unixgram://" UNIX_ADDRESS_STREAM_SCHEME = "unixstream://" +WINDOWS_NAMEDPIPE_SCHEME = "\\\\.\\pipe\\" # Buffering-related values (in seconds) DEFAULT_BUFFERING_FLUSH_INTERVAL = 0.3 @@ -180,12 +186,19 @@ def __init__( >>> statsd = DogStatsd() + :envvar DD_DOGSTATSD_URL: the connection information for the dogstatsd server. + If set, it overrides the default values. + Example for UDP url: `DD_DOGSTATSD_URL=udp://localhost:8125` + Example for UDS: `DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket` + Windows named pipes are currently unsupported. + :type DD_DOGSTATSD_URL: string + :envvar DD_AGENT_HOST: the host of the DogStatsd server. - If set, it overrides default value. + If set, it overrides default value. DD_DOGSTATSD_URL takes precedence over this value. :type DD_AGENT_HOST: string :envvar DD_DOGSTATSD_PORT: the port of the DogStatsd server. - If set, it overrides default value. + If set, it overrides default value. DD_DOGSTATSD_URL takes precedence over this value. :type DD_DOGSTATSD_PORT: integer :envvar DATADOG_TAGS: Tags to attach to every metric reported by dogstatsd client. @@ -353,22 +366,8 @@ def __init__( # Check for deprecated option if max_buffer_size is not None: log.warning("The parameter max_buffer_size is now deprecated and is not used anymore") - # Check host and port env vars - agent_host = os.environ.get("DD_AGENT_HOST") - if agent_host and host == DEFAULT_HOST: - host = agent_host - dogstatsd_port = os.environ.get("DD_DOGSTATSD_PORT") - if dogstatsd_port and port == DEFAULT_PORT: - try: - port = int(dogstatsd_port) - except ValueError: - log.warning( - "Port number provided in DD_DOGSTATSD_PORT env var is not an integer: \ - %s, using %s as port number", - dogstatsd_port, - port, - ) + host, port, socket_path = self._parse_env_connection_overrides(host, port, socket_path) # Assuming environment variables always override telemetry_host = os.environ.get("DD_TELEMETRY_HOST", telemetry_host) @@ -568,6 +567,74 @@ def disable_telemetry(self): def enable_telemetry(self): self._telemetry = True + def _parse_env_connection_overrides(self, host, port, socket_path): + dogstatsd_url = os.environ.get("DD_DOGSTATSD_URL") + + if ( + host == DEFAULT_HOST + and port == DEFAULT_PORT + and socket_path is None + and dogstatsd_url is not None + ): + parsed = urlparse(dogstatsd_url) + # If all values are defaults, prefer DD_DOGSTATSD_URL if present. + if parsed.scheme == "unix": + log.debug( + "Found a DD_DOGSTATSD_URL matching the uds syntax, " + "setting socket path %s.", dogstatsd_url + ) + return host, port, dogstatsd_url + + elif dogstatsd_url.startswith(WINDOWS_NAMEDPIPE_SCHEME): + log.debug( + "DD_DOGSTATSD_URL is configured to utilize a windows named pipe, " + "which is not currently supported by datadogpy. Falling back to " + "alternate connection identifiers." + ) + + elif parsed.scheme == "udp": + try: + p_port = parsed.port + # Python 2 doesn't automatically perform bounds checking on the port + if p_port is None or p_port < 0 or p_port > 65535: + log.debug("Invalid port number provided, reverting to default port") + p_port = DEFAULT_PORT + except ValueError: + log.debug("Invalid port number provided, reverting to default port") + p_port = DEFAULT_PORT + + log.debug( + "Found a DD_DOGSTATSD_URL matching the udp sytnax, " + "setting host and port %s:%d.", parsed.hostname, p_port + ) + + return parsed.hostname, p_port, socket_path + else: + log.debug( + "Unable to parse DD_DOGSTATSD_URL, did you remember to prefix the url " + "with 'unix://' or 'udp://'? Falling back to alternate " + "connection identifiers." + ) + + # We either have some non-default values or no DD_DOGSTATSD_URL + # Check host and port env vars + agent_host = os.environ.get("DD_AGENT_HOST") + if agent_host and host == DEFAULT_HOST: + host = agent_host + + dogstatsd_port = os.environ.get("DD_DOGSTATSD_PORT") + if dogstatsd_port and port == DEFAULT_PORT: + try: + port = int(dogstatsd_port) + except ValueError: + log.warning( + "Port number provided in DD_DOGSTATSD_PORT env var is not an integer: \ + %s, using %s as port number", + dogstatsd_port, + port, + ) + return host, port, socket_path + # Note: Invocations of this method should be thread-safe def _start_flush_thread(self): if self._disable_aggregation and self.disable_buffering: diff --git a/doc/source/index.rst b/doc/source/index.rst index c8a64cb4a..d0fcc1ca9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -40,6 +40,15 @@ Here's an example where the statsd host and port are configured as well:: ) +If statsd_host and statsd_port are left at their default values and no socket_path alternative is supplied, +the DD_DOGSTATSD_URL environment variable, if it exists, will be used to determine the connection +information. This must be a URL that start with either `udp://` (to connect using UDP) or with `unix://` +(to use a Unix Domain Socket). + +* Example for UDP url: `DD_DOGSTATSD_URL=udp://localhost:8125` +* Example for UDS: `DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket` + + .. autofunction:: datadog.initialize diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 7ab43f508..d3dbf32b9 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -30,7 +30,7 @@ # Datadog libraries from datadog import initialize, statsd from datadog import __version__ as version -from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DogStatsd, MIN_SEND_BUFFER_SIZE, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_OPTIMAL_PAYLOAD_LENGTH from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k from tests.util.contextmanagers import preserve_environment_variable, EnvVars @@ -283,7 +283,7 @@ def test_initialization(self): self.assertIsNone(statsd.host) self.assertIsNone(statsd.port) - def test_dogstatsd_initialization_with_env_vars(self): + def test_dogstatsd_initialization_with_env_vars_agent_host(self): """ Dogstatsd can retrieve its config from env vars when not provided in constructor. @@ -299,6 +299,54 @@ def test_dogstatsd_initialization_with_env_vars(self): self.assertEqual(dogstatsd.host, "myenvvarhost") self.assertEqual(dogstatsd.port, 4321) + + def test_dogstatsd_initialization_with_env_vars_dogstatsd_url(self): + """ + Dogstatsd can retrieve its config from env vars when + not provided in constructor. + """ + # Setup UDP + with preserve_environment_variable('DD_DOGSTATSD_URL'): + os.environ['DD_DOGSTATSD_URL'] = 'udp://myenvvarhost:4321' + dogstatsd = DogStatsd() + + # Assert + self.assertEqual(dogstatsd.host, "myenvvarhost") + self.assertEqual(dogstatsd.port, 4321) + self.assertEqual(dogstatsd.socket_path, None) + + # Test UDS + with preserve_environment_variable('DD_DOGSTATSD_URL'): + os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock' + dogstatsd = DogStatsd() + self.assertEqual(dogstatsd.socket_path, 'unix:///hello/world.sock') + self.assertEqual(dogstatsd.host, None) + self.assertEqual(dogstatsd.port, None) + + # Test non-default host + with preserve_environment_variable('DD_DOGSTATSD_URL'): + os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock' + dogstatsd = DogStatsd(host="myhost") + self.assertEqual(dogstatsd.socket_path, None) + self.assertEqual(dogstatsd.host, 'myhost') + self.assertEqual(dogstatsd.port, DEFAULT_PORT) + + # Test non-default port + with preserve_environment_variable('DD_DOGSTATSD_URL'): + os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock' + dogstatsd = DogStatsd(port=8240) + self.assertEqual(dogstatsd.socket_path, None) + self.assertEqual(dogstatsd.host, DEFAULT_HOST) + self.assertEqual(dogstatsd.port, 8240) + + # Test non-default socket_path + with preserve_environment_variable('DD_DOGSTATSD_URL'): + os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock' + dogstatsd = DogStatsd(socket_path='unix:///var/run/datadog/dsd.sock') + self.assertEqual(dogstatsd.socket_path, 'unix:///var/run/datadog/dsd.sock') + self.assertEqual(dogstatsd.host, None) + self.assertEqual(dogstatsd.port, None) + def test_initialization_closes_socket(self): statsd.socket = FakeSocket() self.assertIsNotNone(statsd.socket)