Add repeat until failure command (#118)

* ✨ Add r command to repeat tests until failure

* ♻️ Extract run summary into its own module

Also, refactor for less string concatentation.

README.md

@@ -18,8 +18,9 @@ which tests should be run with a few keystrokes.
 It allows you to easily switch between running all tests, stale tests, or failed
 tests. Or, you can run only the tests whose filenames contain a substring. You
 can also control which tags are included or excluded, modify the maximum number
-of failures allowed, specify the test seed to use, and toggle tracing on and
-off. Includes an optional "watch mode" which runs tests after every file change.
+of failures allowed, repeat the test suite until a failure occurs, specify the
+test seed to use, and toggle tracing on and off. Includes an optional "watch
+mode" which runs tests after every file change.
 
 ## Installation
 
@@ -118,6 +119,10 @@ will run.
   e.g. `test/my_project/my_test.exs`, `test/my_project/my_test.exs:12:24` or
   `my`.
 - `q`: Exit the program. (Can also use `Ctrl-D`.)
+- `r <count>`: (Elixir 1.17.0 and later) Run tests up to <count> times until a
+  failure occurs (equivalent to the `--repeat-until-failure` option of `mix
+test`).
+- `r`: (Elixir 1.17.0 and later) Clear the "repeat-until-failure" count.
 - `s`: Run only test files that reference modules that have changed since the
   last run (equivalent to the `--stale` option of `mix test`).
 - `t`: Turn test tracing on or off (equivalent to the `--trace` option of `mix

lib/mix/tasks/test/interactive.ex

@@ -6,9 +6,9 @@ defmodule Mix.Tasks.Test.Interactive do
   `mix test.interactive` allows you to easily switch between running all tests,
   stale tests, or failed tests. Or, you can run only the tests whose filenames
   contain a substring. You can also control which tags are included or excluded,
-  modify the maximum number of failures allowed, specify the test seed to use,
-  and toggle tracing on and off. Includes an optional "watch mode" which runs
-  tests after every file change.
+  modify the maximum number of failures allowed, repeat the test suite until a
+  failure occurs, specify the test seed to use, and toggle tracing on and off.
+  Includes an optional "watch mode" which runs tests after every file change.
 
   ## Usage
 
@@ -94,6 +94,10 @@ defmodule Mix.Tasks.Test.Interactive do
     e.g. `test/my_project/my_test.exs`, `test/my_project/my_test.exs:12:24` or
     `my`.
   - `q`: Exit the program. (Can also use `Ctrl-D`.)
+  - `r <count>`: (Elixir 1.17.0 and later) Run tests up to <count> times until a
+    failure occurs (equivalent to the `--repeat-until-failure` option of `mix
+    test`).
+  - `r`: (Elixir 1.17.0 and later) Clear the "repeat-until-failure" count.
   - `s`: Run only test files that reference modules that have changed since the
     last run (equivalent to the `--stale` option of `mix test`).
   - `t`: Turn test tracing on or off (equivalent to the `--trace` option of `mix

lib/mix_test_interactive/command/repeat_until_failure.ex

@@ -0,0 +1,30 @@
+defmodule MixTestInteractive.Command.RepeatUntilFailure do
+  @moduledoc """
+  Specify or clear the number of repetitions for running until failure.
+
+  Runs the tests repeatedly until failure or until the specified number of runs.
+  If not provided, the count is cleared and the tests will run just once as
+  usual.
+
+  Corresponds to `mix test --repeat-until-failure <count>`.
+
+  This option is only available in `mix test` in Elixir 1.17.0 and later.
+  """
+  use MixTestInteractive.Command, command: "r", desc: "set or clear the repeat-until-failure count"
+
+  alias MixTestInteractive.Command
+  alias MixTestInteractive.Settings
+
+  @impl Command
+  def name, do: "r [<count>]"
+
+  @impl Command
+  def run([], %Settings{} = settings) do
+    {:ok, Settings.clear_repeat_count(settings)}
+  end
+
+  @impl Command
+  def run([count], %Settings{} = settings) do
+    {:ok, Settings.with_repeat_count(settings, count)}
+  end
+end

lib/mix_test_interactive/command_line_parser.ex

@@ -168,6 +168,7 @@ defmodule MixTestInteractive.CommandLineParser do
     {includes, mix_test_opts} = Keyword.pop_values(mix_test_opts, :include)
     {only, mix_test_opts} = Keyword.pop_values(mix_test_opts, :only)
     {max_failures, mix_test_opts} = Keyword.pop(mix_test_opts, :max_failures)
+    {repeat_count, mix_test_opts} = Keyword.pop(mix_test_opts, :repeat_until_failure)
     {seed, mix_test_opts} = Keyword.pop(mix_test_opts, :seed)
     {stale?, mix_test_opts} = Keyword.pop(mix_test_opts, :stale, false)
     {trace?, mix_test_opts} = Keyword.pop(mix_test_opts, :trace, false)
@@ -181,6 +182,7 @@ defmodule MixTestInteractive.CommandLineParser do
       max_failures: max_failures && to_string(max_failures),
       only: only,
       patterns: patterns,
+      repeat_count: repeat_count && to_string(repeat_count),
       seed: seed && to_string(seed),
       stale?: no_patterns? && !failed? && stale?,
       tracing?: trace?,

lib/mix_test_interactive/command_processor.ex

@@ -13,6 +13,7 @@ defmodule MixTestInteractive.CommandProcessor do
   alias MixTestInteractive.Command.Only
   alias MixTestInteractive.Command.Pattern
   alias MixTestInteractive.Command.Quit
+  alias MixTestInteractive.Command.RepeatUntilFailure
   alias MixTestInteractive.Command.RunTests
   alias MixTestInteractive.Command.Seed
   alias MixTestInteractive.Command.Stale
@@ -32,6 +33,7 @@ defmodule MixTestInteractive.CommandProcessor do
     Only,
     Pattern,
     Quit,
+    RepeatUntilFailure,
     RunTests,
     Seed,
     Stale,

lib/mix_test_interactive/interactive_mode.ex

@@ -14,6 +14,7 @@ defmodule MixTestInteractive.InteractiveMode do
   alias MixTestInteractive.CommandProcessor
   alias MixTestInteractive.Config
   alias MixTestInteractive.Runner
+  alias MixTestInteractive.RunSummary
   alias MixTestInteractive.Settings
 
   @type option :: {:config, Config.t()} | {:name | String.t()}
@@ -116,7 +117,7 @@ defmodule MixTestInteractive.InteractiveMode do
     IO.puts("")
 
     settings
-    |> Settings.summary()
+    |> RunSummary.from_settings()
     |> IO.puts()
   end
 

lib/mix_test_interactive/run_summary.ex

@@ -0,0 +1,71 @@
+defmodule MixTestInteractive.RunSummary do
+  @moduledoc false
+  alias MixTestInteractive.Settings
+
+  @doc """
+  Return a text summary of the current interactive mode settings.
+  """
+  @spec from_settings(Settings.t()) :: String.t()
+  def from_settings(%Settings{} = settings) do
+    [&base_summary/1, &all_tag_filters/1, &max_failures/1, &repeat_count/1, &seed/1, &tracing/1]
+    |> Enum.flat_map(fn fun -> List.wrap(fun.(settings)) end)
+    |> Enum.join("\n")
+  end
+
+  defp all_tag_filters(%Settings{} = settings) do
+    Enum.reject(
+      [
+        tag_filters("Excluding tags", settings.excludes),
+        tag_filters("Including tags", settings.includes),
+        tag_filters("Only tags", settings.only)
+      ],
+      &is_nil/1
+    )
+  end
+
+  defp base_summary(%Settings{} = settings) do
+    cond do
+      settings.failed? ->
+        "Ran only failed tests"
+
+      settings.stale? ->
+        "Ran only stale tests"
+
+      !Enum.empty?(settings.patterns) ->
+        "Ran all test files matching #{Enum.join(settings.patterns, ", ")}"
+
+      true ->
+        "Ran all tests"
+    end
+  end
+
+  defp max_failures(%Settings{max_failures: nil}), do: nil
+
+  defp max_failures(%Settings{} = settings) do
+    "Max failures: #{settings.max_failures}"
+  end
+
+  defp repeat_count(%Settings{repeat_count: nil}), do: nil
+
+  defp repeat_count(%Settings{} = settings) do
+    "Repeat until failure: #{settings.repeat_count}"
+  end
+
+  def seed(%Settings{seed: nil}), do: nil
+
+  def seed(%Settings{} = settings) do
+    "Seed: #{settings.seed}"
+  end
+
+  defp tracing(%Settings{tracing?: false}), do: nil
+
+  defp tracing(%Settings{}) do
+    "Tracing: ON"
+  end
+
+  defp tag_filters(_label, []), do: nil
+
+  defp tag_filters(label, tags) do
+    label <> ": " <> inspect(tags)
+  end
+end

lib/mix_test_interactive/settings.ex

@@ -22,6 +22,7 @@ defmodule MixTestInteractive.Settings do
     field :max_failures, String.t()
     field :only, [String.t()], default: []
     field :patterns, [String.t()], default: []
+    field :repeat_count, String.t()
     field :seed, String.t()
     field :stale?, boolean(), default: false
     field :tracing?, boolean(), default: false
@@ -69,6 +70,15 @@ defmodule MixTestInteractive.Settings do
     %{settings | only: []}
   end
 
+  @doc """
+  Update settings to run tests only once, clearing any repeat-until-failure
+  count.
+  """
+  @spec clear_repeat_count(t()) :: t()
+  def clear_repeat_count(%__MODULE__{} = settings) do
+    %{settings | repeat_count: nil}
+  end
+
   @doc """
   Update settings to run tests with a random seed, clearing any specified seed.
   """
@@ -160,6 +170,7 @@ defmodule MixTestInteractive.Settings do
     with_seed
     |> append_tag_filters(settings)
     |> append_max_failures(settings)
+    |> append_repeat_count(settings)
     |> append_tracing(settings)
   end
 
@@ -171,6 +182,12 @@ defmodule MixTestInteractive.Settings do
     summary <> "\nMax failures: #{settings.max_failures}"
   end
 
+  defp append_repeat_count(summary, %__MODULE__{repeat_count: nil}), do: summary
+
+  defp append_repeat_count(summary, %__MODULE__{} = settings) do
+    summary <> "\nRepeat until failure: #{settings.repeat_count}"
+  end
+
   defp append_tag_filters(summary, %__MODULE__{} = settings) do
     [
       summary,
@@ -250,6 +267,16 @@ defmodule MixTestInteractive.Settings do
     %{settings | only: only}
   end
 
+  @doc """
+  Update settings to run tests <count> times until failure.
+
+  Corresponds to `mix test --repeat-until-failure <count>`.
+  """
+  @spec with_repeat_count(t(), String.t()) :: t()
+  def with_repeat_count(%__MODULE__{} = settings, count) do
+    %{settings | repeat_count: count}
+  end
+
   @doc """
   Update settings to run tests with a specific seed.
 
@@ -302,6 +329,9 @@ defmodule MixTestInteractive.Settings do
     Enum.flat_map(only, &["--only", &1])
   end
 
+  defp opts_from_single_setting({:repeat_count, nil}), do: []
+  defp opts_from_single_setting({:repeat_count, count}), do: ["--repeat-until-failure", count]
+
   defp opts_from_single_setting({:seed, nil}), do: []
   defp opts_from_single_setting({:seed, seed}), do: ["--seed", seed]
 

test/mix_test_interactive/command_line_parser_test.exs

@@ -258,6 +258,12 @@ defmodule MixTestInteractive.CommandLineParserTest do
       assert settings.initial_cli_args == ["--color", "--raise"]
     end
 
+    test "extracts repeat-until-failure from arguments" do
+      {:ok, %{settings: settings}} = CommandLineParser.parse(["--color", "--repeat-until-failure", "1000", "--raise"])
+      assert settings.repeat_count == "1000"
+      assert settings.initial_cli_args == ["--color", "--raise"]
+    end
+
     test "extracts seed from arguments" do
       {:ok, %{settings: settings}} = CommandLineParser.parse(["--color", "--seed", "5432", "--raise"])
       assert settings.seed == "5432"

test/mix_test_interactive/command_processor_test.exs

@@ -107,6 +107,20 @@ defmodule MixTestInteractive.CommandProcessorTest do
       assert {:ok, ^expected} = process_command("p second", first_config)
     end
 
+    test "r <count> sets the repeat until failure count" do
+      settings = %Settings{}
+      expected = Settings.with_repeat_count(settings, "4200")
+
+      assert {:ok, ^expected} = process_command("r 4200", settings)
+    end
+
+    test "r with no count clears the repeat until failure count" do
+      {:ok, settings} = process_command("r 1000", %Settings{})
+      expected = Settings.clear_repeat_count(settings)
+
+      assert {:ok, ^expected} = process_command("r", settings)
+    end
+
     test "s runs only stale tests" do
       settings = %Settings{}
       expected = Settings.only_stale(settings)

test/mix_test_interactive/end_to_end_test.exs

@@ -73,6 +73,16 @@ defmodule MixTestInteractive.EndToEndTest do
     assert_ran_tests()
   end
 
+  test "repeat until failure workflow", %{pid: pid} do
+    assert_ran_tests()
+
+    assert :ok = InteractiveMode.process_command(pid, "r 1000")
+    assert_ran_tests(["--repeat-until-failure", "1000"])
+
+    assert :ok = InteractiveMode.process_command(pid, "r")
+    assert_ran_tests()
+  end
+
   test "seed workflow", %{pid: pid} do
     assert_ran_tests()