[SDTEST-437] Auto test retries for minitest

lib/datadog/ci/contrib/minitest/runner.rb

@@ -28,6 +28,19 @@ def init_plugins(*args)
               test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end
 
+            def run_one_method(klass, method_name)
+              return super unless datadog_configuration[:enabled]
+
+              result = nil
+              # retries here
+              test_retries_component.with_retries do |test_finished_callback|
+                Thread.current[:__dd_retry_callback] = test_finished_callback
+
+                result = super
+              end
+              result
+            end
+
             private
 
             def datadog_configuration
@@ -37,6 +50,10 @@ def datadog_configuration
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end
+
+            def test_retries_component
+              Datadog.send(:components).test_retries
+            end
           end
         end
       end

lib/datadog/ci/contrib/minitest/test.rb

@@ -50,7 +50,7 @@ def after_teardown
               test_span = test_visibility_component.active_test
               return super unless test_span
 
-              finish_with_result(test_span, result_code)
+              finish_with_result(test_span, result_code, Thread.current[:__dd_retry_callback])
               if Helpers.parallel?(self.class)
                 finish_with_result(test_span.test_suite, result_code)
               end
@@ -60,7 +60,7 @@ def after_teardown
 
             private
 
-            def finish_with_result(span, result_code)
+            def finish_with_result(span, result_code, callback = nil)
               return unless span
 
               case result_code
@@ -71,6 +71,9 @@ def finish_with_result(span, result_code)
               when "S"
                 span.skipped!(reason: failure.message)
               end
+
+              callback.call(span) if callback
+
               span.finish
             end
 

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -948,4 +948,123 @@ def test_with_background_thread
       )
     end
   end
+
+  context "with flaky test and test retries enabled" do
+    include_context "CI mode activated" do
+      let(:integration_name) { :minitest }
+
+      let(:flaky_test_retries_enabled) { true }
+    end
+
+    before do
+      Minitest.run([])
+    end
+
+    before(:context) do
+      Thread.current[:dd_coverage_collector] = nil
+
+      Minitest::Runnable.reset
+
+      class FlakyTestSuite < Minitest::Test
+        @@max_flaky_test_failures = 4
+        @@flaky_test_failures = 0
+
+        def test_passed
+          assert true
+        end
+
+        def test_flaky
+          if @@flaky_test_failures < @@max_flaky_test_failures
+            @@flaky_test_failures += 1
+            assert 1 + 1 == 3
+          else
+            assert 1 + 1 == 2
+          end
+        end
+      end
+    end
+
+    it "retries flaky test" do
+      # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans
+      expect(test_spans).to have(6).items
+
+      failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
+      expect(failed_spans).to have(4).items # see steps.rb
+      expect(passed_spans).to have(2).items
+
+      test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
+      expect(test_spans_by_test_name["test_flaky"]).to have(5).items
+
+      # count how many spans were marked as retries
+      retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+      expect(retries_count).to eq(4)
+
+      expect(test_spans_by_test_name["test_passed"]).to have(1).item
+
+      expect(test_suite_spans).to have(1).item
+      expect(test_suite_spans.first).to have_pass_status
+
+      expect(test_session_span).to have_pass_status
+    end
+  end
+
+  context "with flaky test and test retries enabled with insufficient max retries" do
+    include_context "CI mode activated" do
+      let(:integration_name) { :minitest }
+
+      let(:flaky_test_retries_enabled) { true }
+      let(:retry_failed_tests_max_attempts) { 3 }
+    end
+
+    before do
+      Minitest.run([])
+    end
+
+    before(:context) do
+      Thread.current[:dd_coverage_collector] = nil
+
+      Minitest::Runnable.reset
+
+      class FlakyTestSuite2 < Minitest::Test
+        @@max_flaky_test_failures = 4
+        @@flaky_test_failures = 0
+
+        def test_passed
+          assert true
+        end
+
+        def test_flaky
+          if @@flaky_test_failures < @@max_flaky_test_failures
+            @@flaky_test_failures += 1
+            assert 1 + 1 == 3
+          else
+            assert 1 + 1 == 2
+          end
+        end
+      end
+    end
+
+    it "retries flaky test" do
+      # 1 initial run of flaky test + 3 retries without success + 1 passing test = 5 spans
+      expect(test_spans).to have(5).items
+
+      failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
+      expect(failed_spans).to have(4).items # see steps.rb
+      expect(passed_spans).to have(1).items
+
+      test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
+      expect(test_spans_by_test_name["test_flaky"]).to have(4).items
+
+      # count how many spans were marked as retries
+      retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+      expect(retries_count).to eq(3)
+
+      expect(test_spans_by_test_name["test_passed"]).to have(1).item
+
+      expect(test_suite_spans).to have(1).item
+      expect(test_suite_spans.first).to have_fail_status
+
+      expect(test_session_span).to have_fail_status
+    end
+  end
 end