Parameterize the exponential backoff

The list of delays passed to Slipstream is now parameterized by a
minimum delay, a maximum and a jitter. The default is to first delay for
a second and then progressively double it until a 60 second delay.

Jitter is specified as a percentage to add. For 0.5 jitter, this means
that what would normally be a 1 second delay might be up to 1.5 seconds.
Similarly, a 10 second delay might be as long as 15 seconds and so on.

The delays are seeded from `:crypto.strong_rand_bytes/1` to help ensure
that there's a good spread across devices. While `:rand.seed/2`'s
defaults may be ok, there was concern since it's seeded by the system
time.

lib/nerves_hub_link/backoff.ex

@@ -0,0 +1,44 @@
+defmodule NervesHubLink.Backoff do
+  @moduledoc """
+  Compute retry backoff intervals used by Slipstream
+  """
+
+  @doc """
+  Produce a list of integer backoff delays with jitter
+
+  The first two parameters are minimum and maximum value. These are expected to
+  be milliseconds, but this function doesn't care. The returned list will start
+  with the minimum value and then double it until it reaches the maximum value.
+
+  The third parameter is the amount of jitter to add to each delay. The value
+  should be between 0 and 1. Zero adds no jitter. A value like 0.25 will add up
+  to 25% of the delay amount.
+  """
+  @spec delay_list(integer(), integer(), number()) :: [integer()]
+  def delay_list(min, max, jitter) when min > 0 and max >= min and jitter >= 0 do
+    seed_rand()
+    calc(min, max, jitter)
+  end
+
+  defp calc(min, max, jitter) when min >= max do
+    [add_jitter(max, jitter)]
+  end
+
+  defp calc(min, max, jitter) do
+    delay = add_jitter(min, jitter)
+    [delay | calc(min * 2, max, jitter)]
+  end
+
+  defp add_jitter(value, jitter) do
+    round(value * (1 + jitter * :rand.uniform()))
+  end
+
+  defp seed_rand() do
+    # `:rand` gets seeded with the system time and counters. To avoid concern
+    # that the seed could be the same across many devices, pull from a pool of
+    # cryptographically secure random numbers.
+    <<x::32>> = :crypto.strong_rand_bytes(4)
+    _ = :rand.seed(:exsss, x)
+    :ok
+  end
+end

lib/nerves_hub_link/configurator.ex

@@ -1,4 +1,5 @@
 defmodule NervesHubLink.Configurator do
+  alias NervesHubLink.Backoff
   alias __MODULE__.{Config, Default}
   require Logger
 
@@ -52,9 +53,17 @@ defmodule NervesHubLink.Configurator do
     # any other items that may have been provided in :socket or
     # :transport_opts keys previously.
     transport_opts = config.socket[:transport_opts] || []
-
     transport_opts = Keyword.put(transport_opts, :socket_opts, config.ssl)
-    socket = Keyword.put(config.socket, :transport_opts, transport_opts)
+
+    socket =
+      config.socket
+      |> Keyword.put(:transport_opts, transport_opts)
+      |> Keyword.put_new_lazy(:reconnect_after_msec, fn ->
+        # Default retry interval
+        # 1 second minimum delay that doubles up to 60 seconds. Up to 50% of
+        # the delay is added to introduce jitter into the retry attempts.
+        Backoff.delay_list(1000, 60000, 0.50)
+      end)
 
     %{config | socket: socket}
   end

lib/nerves_hub_link/socket.ex

@@ -52,7 +52,7 @@ defmodule NervesHubLink.Socket do
       mint_opts: [protocols: [:http1], transport_opts: config.ssl],
       uri: config.socket[:url],
       rejoin_after_msec: [@rejoin_after],
-      reconnect_after_msec: backoff()
+      reconnect_after_msec: config.socket[:reconnect_after_msec]
     ]
 
     socket =
@@ -297,13 +297,4 @@ defmodule NervesHubLink.Socket do
     :ok = GenServer.stop(iex, 10_000)
     assign(socket, iex_pid: nil)
   end
-
-  # Produces a jittered list of 11 that grows exponentially from less than a second to about 60 secs
-  defp backoff() do
-    jitter = Enum.random(0..500)
-
-    for i <- 1..11 do
-      round(:math.exp(i)) + jitter
-    end
-  end
 end

test/nerves_hub_link/backoff_test.exs

@@ -0,0 +1,31 @@
+defmodule NervesHubLink.BackoffTest do
+  use ExUnit.Case
+  alias NervesHubLink.Backoff
+
+  test "no jitter" do
+    assert Backoff.delay_list(1000, 60000, 0) == [1000, 2000, 4000, 8000, 16000, 32000, 60000]
+  end
+
+  test "some jitter" do
+    # Check that the jitter averages out to the expected value after a lot
+    # of runs.
+    runs = for _ <- 1..1000, do: backoff_average_jitter(1000, 60000, 0.25)
+    run_average = average(runs)
+
+    # The average of all of the runs should be around half 0.25
+    assert_in_delta run_average, 0.125, 0.04
+  end
+
+  defp backoff_average_jitter(low, high, jitter) do
+    zero_jitter = Backoff.delay_list(low, high, 0)
+    test_jitter = Backoff.delay_list(low, high, jitter)
+
+    Enum.zip(zero_jitter, test_jitter)
+    |> Enum.map(fn {z, t} -> abs((t - z) / z) end)
+    |> average()
+  end
+
+  defp average(numbers) do
+    Enum.sum(numbers) / length(numbers)
+  end
+end