add retries with exponential backoff on network errors

Steepfile

@@ -15,6 +15,7 @@ target :lib do
   library "securerandom"
   library "tmpdir"
   library "fileutils"
+  library "socket"
 
   repo_path "vendor/rbs"
   library "ddtrace"

lib/datadog/ci/transport/http.rb

@@ -4,6 +4,7 @@
 require "datadog/core/transport/http/adapters/net"
 require "datadog/core/transport/http/env"
 require "datadog/core/transport/request"
+require "socket"
 
 require_relative "gzip"
 require_relative "../ext/transport"
@@ -20,6 +21,8 @@ class HTTP
           :compress
 
         DEFAULT_TIMEOUT = 30
+        MAX_RETRIES = 3
+        INITIAL_BACKOFF = 1
 
         def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress: false)
           @host = host
@@ -29,7 +32,7 @@ def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress:
           @compress = compress.nil? ? false : compress
         end
 
-        def request(path:, payload:, headers:, verb: "post")
+        def request(path:, payload:, headers:, verb: "post", retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
           if compress
             headers[Ext::Transport::HEADER_CONTENT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
             payload = Gzip.compress(payload)
@@ -41,9 +44,7 @@ def request(path:, payload:, headers:, verb: "post")
           end
 
           response = ResponseDecorator.new(
-            adapter.call(
-              build_env(path: path, payload: payload, headers: headers, verb: verb)
-            )
+            perform_http_call(path: path, payload: payload, headers: headers, verb: verb, retries: retries, backoff: backoff)
           )
 
           Datadog.logger.debug do
@@ -55,6 +56,25 @@ def request(path:, payload:, headers:, verb: "post")
 
         private
 
+        def perform_http_call(path:, payload:, headers:, verb:, retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
+          adapter.call(
+            build_env(path: path, payload: payload, headers: headers, verb: verb)
+          )
+        rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, SocketError, Net::HTTPBadResponse => e
+          Datadog.logger.debug("Failed to send request with #{e} (#{e.message})")
+
+          if retries.positive?
+            sleep(backoff)
+
+            perform_http_call(
+              path: path, payload: payload, headers: headers, verb: verb, retries: retries - 1, backoff: backoff * 2
+            )
+          else
+            Datadog.logger.error("Failed to send request after #{MAX_RETRIES} retries")
+            raise e
+          end
+        end
+
         def build_env(path:, payload:, headers:, verb:)
           env = Datadog::Core::Transport::HTTP::Env.new(
             Datadog::Core::Transport::Request.new

sig/datadog/ci/transport/http.rbs

@@ -14,17 +14,21 @@ module Datadog
         attr_reader compress: bool
 
         DEFAULT_TIMEOUT: 30
+        MAX_RETRIES: 3
+        INITIAL_BACKOFF: 1
 
         def initialize: (host: String, ?port: Integer?, ?ssl: bool, ?timeout: Integer, ?compress: bool) -> void
 
-        def request: (?verb: String, payload: String, headers: Hash[String, String], path: String) -> ResponseDecorator
+        def request: (?verb: String, payload: String, headers: Hash[String, String], path: String, ?retries: Integer, ?backoff: Integer) -> ResponseDecorator
 
         private
 
         def adapter: () -> Datadog::Core::Transport::HTTP::Adapters::Net
 
         def build_env: (payload: String, headers: Hash[String, String], path: String, verb: String) -> Datadog::Core::Transport::HTTP::Env
 
+        def perform_http_call: (payload: String, headers: Hash[String, String], path: String, verb: String, ?retries: Integer, ?backoff: Integer) -> Datadog::Core::Transport::Response
+
         class AdapterSettings
           attr_reader hostname: String
           attr_reader port: Integer?

spec/datadog/ci/transport/http_spec.rb

@@ -145,5 +145,32 @@
         expect(response.code).to eq(200)
       end
     end
+
+    context "when request fails" do
+      let(:request_options) { {backoff: 0} }
+
+      context "when succeeds after retries" do
+        before do
+          expect(adapter).to receive(:call).and_raise(Errno::ECONNRESET).exactly(described_class::MAX_RETRIES).times
+          expect(adapter).to receive(:call).and_return(http_response)
+        end
+
+        it "produces a response" do
+          is_expected.to be_a_kind_of(described_class::ResponseDecorator)
+
+          expect(response.code).to eq(200)
+        end
+      end
+
+      context "when retries are exhausted" do
+        before do
+          expect(adapter).to receive(:call).and_raise(Errno::ECONNRESET).exactly(described_class::MAX_RETRIES + 1).times
+        end
+
+        it "raises" do
+          expect { response }.to raise_error(Errno::ECONNRESET)
+        end
+      end
+    end
   end
 end