rewrite cucumber's test retries feature

lib/datadog/ci/contrib/cucumber/filter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Cucumber
+        class Filter < ::Cucumber::Core::Filter.new(:configuration)
+          def test_case(test_case)
+            test_retries_component.reset_retries! unless test_case_seen[test_case]
+
+            test_case_seen[test_case] = true
+            configuration.on_event(:test_case_finished) do |event|
+              next unless retry_required?(test_case, event)
+
+              test_case.describe_to(receiver)
+            end
+
+            super
+          end
+
+          private
+
+          def retry_required?(test_case, event)
+            return false unless event.test_case == test_case
+
+            test_retries_component.should_retry?
+          end
+
+          def test_case_seen
+            @test_case_seen ||= Hash.new { |h, k| h[k] = false }
+          end
+
+          def test_retries_component
+            @test_retries_component ||= Datadog.send(:components).test_retries
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -18,10 +18,24 @@ module InstanceMethods
 
             def formatters
               existing_formatters = super
-              @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
+              @datadog_formatter ||= Formatter.new(@configuration)
               [@datadog_formatter] + existing_formatters
             end
 
+            def filters
+              require_relative "filter"
+
+              filters_list = super
+              datadog_filter = Filter.new(@configuration)
+              unless @configuration.dry_run?
+                # insert our filter the pre-last position because Cucumber::Filters::PrepareWorld must be the last one
+                # see:
+                # https://github.com/cucumber/cucumber-ruby/blob/58dd8f12c0ac5f4e607335ff2e7d385c1ed25899/lib/cucumber/runtime.rb#L266
+                filters_list.insert(-2, datadog_filter)
+              end
+              filters_list
+            end
+
             def begin_scenario(test_case)
               if Datadog::CI.active_test&.skipped_by_itr?
                 raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON

lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -3,7 +3,6 @@
 require "datadog/tracing/contrib/patcher"
 
 require_relative "instrumentation"
-require_relative "configuration_override"
 
 module Datadog
   module CI
@@ -21,7 +20,6 @@ def target_version
 
           def patch
             ::Cucumber::Runtime.include(Instrumentation)
-            ::Cucumber::Configuration.include(ConfigurationOverride)
           end
         end
       end

lib/datadog/ci/test_retries/component.rb

@@ -53,15 +53,15 @@ def configure(library_settings, test_session)
         end
 
         def with_retries(&block)
-          self.current_retry_driver = nil
+          reset_retries!
 
           loop do
             yield
 
-            break unless current_retry_driver&.should_retry?
+            break unless should_retry?
           end
         ensure
-          self.current_retry_driver = nil
+          reset_retries!
         end
 
         def build_driver(test_span)
@@ -89,6 +89,15 @@ def record_test_span_duration(tracer_span)
           current_retry_driver&.record_duration(tracer_span.duration)
         end
 
+        # this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method
+        def reset_retries!
+          self.current_retry_driver = nil
+        end
+
+        def should_retry?
+          !!current_retry_driver&.should_retry?
+        end
+
         private
 
         def current_retry_driver

lib/datadog/ci/test_retries/null_component.rb

@@ -6,13 +6,7 @@ module Datadog
   module CI
     module TestRetries
       class NullComponent < Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
-
         def initialize
-          # enabled only by remote settings
-          @retry_failed_tests_enabled = false
-          @retry_failed_tests_max_attempts = 0
-          @retry_failed_tests_total_limit = 0
         end
 
         def configure(library_settings)
@@ -22,6 +16,13 @@ def with_retries(&block)
           no_action = proc {}
           yield no_action
         end
+
+        def reset_retries!
+        end
+
+        def should_retry?
+          false
+        end
       end
     end
   end

sig/datadog/ci/contrib/cucumber/filter.rbs

@@ -0,0 +1,23 @@
+module Datadog
+  module CI
+    module Contrib
+      module Cucumber
+        class Filter < ::Cucumber::Core::Filter
+          @test_case_seen: untyped
+
+          @test_retries_component: untyped
+
+          def test_case: (untyped test_case) -> untyped
+
+          private
+
+          def retry_required?: (untyped test_case, untyped event) -> (false | untyped)
+
+          def test_case_seen: () -> untyped
+
+          def test_retries_component: () -> untyped
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/test_retries/component.rbs

@@ -20,6 +20,10 @@ module Datadog
 
         def record_test_span_duration: (Datadog::Tracing::SpanOperation span) -> void
 
+        def reset_retries!: () -> void
+
+        def should_retry?: () -> bool
+
         private
 
         def current_retry_driver: () -> Datadog::CI::TestRetries::Driver::Base?

spec/datadog/ci/contrib/cucumber/features/flaky.feature

@@ -3,6 +3,10 @@ Feature: When I have flaky test
     When flaky
     Then datadog
 
+  Scenario: another flaky scenario
+    When flaky
+    Then datadog
+
   Scenario: this scenario just passes
     When datadog
     Then datadog

spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb

@@ -43,5 +43,7 @@
   if flaky_test_executions < max_flaky_test_failures
     flaky_test_executions += 1
     raise "Flaky test failure"
+  else
+    flaky_test_executions = 0
   end
 end

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -3,7 +3,7 @@
 require "cucumber"
 require "securerandom"
 
-RSpec.describe "Cucumber formatter" do
+RSpec.describe "Cucumber instrumentation" do
   let(:cucumber_features_root) { File.join(__dir__, "features") }
   let(:enable_retries) { false }
   let(:single_test_retries_count) { 5 }
@@ -477,19 +477,24 @@
     end
 
     it "retries the test several times and correctly tracks result of every invocation" do
-      # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans
-      expect(test_spans).to have(6).items
+      # 1 initial run of flaky test + 4 retries until pass + 1 passing test + 1 other flaky + 4 retries until pass = 11 spans
+      expect(test_spans).to have(11).items
 
       failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-      expect(failed_spans).to have(4).items # see steps.rb
-      expect(passed_spans).to have(2).items
+      expect(failed_spans).to have(8).items # see steps.rb
+      expect(passed_spans).to have(3).items
 
       test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
       expect(test_spans_by_test_name["very flaky scenario"]).to have(5).items
+      expect(test_spans_by_test_name["another flaky scenario"]).to have(5).items
 
       # count how many spans were marked as retries
       retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
-      expect(retries_count).to eq(4)
+      expect(retries_count).to eq(8)
+
+      # count how many spans were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(0)
 
       expect(test_spans_by_test_name["this scenario just passes"]).to have(1).item
 
@@ -505,19 +510,24 @@
     let(:feature_file_to_run) { "flaky.feature" }
 
     it "retries the test several times and correctly tracks result of every invocation" do
-      # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans
-      expect(test_spans).to have(6).items
+      # 1 initial run of flaky test + 4 retries until pass + 1 passing test + 1 other flaky + 4 retries = 11 spans
+      expect(test_spans).to have(11).items
 
       failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-      expect(failed_spans).to have(4).items # see steps.rb
-      expect(passed_spans).to have(2).items
+      expect(failed_spans).to have(8).items # see steps.rb
+      expect(passed_spans).to have(3).items
 
       test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
       expect(test_spans_by_test_name["very flaky scenario"]).to have(5).items
+      expect(test_spans_by_test_name["another flaky scenario"]).to have(5).items
 
       # count how many spans were marked as retries
       retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
-      expect(retries_count).to eq(4)
+      expect(retries_count).to eq(8)
+
+      # count how many spans were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(0)
 
       expect(test_spans_by_test_name["this scenario just passes"]).to have(1).item
 
@@ -532,13 +542,17 @@
       let(:expected_test_run_code) { 2 }
 
       it "retries the test once" do
-        # 1 initial run of flaky test + 1 retry + 1 passing = 3 spans
-        expect(test_spans).to have(3).items
+        # 1 initial run of flaky test + 1 retry + 1 passing + 1 other flaky + 1 retry = 5 spans
+        expect(test_spans).to have(5).items
         retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
-        expect(retries_count).to eq(1)
+        expect(retries_count).to eq(2)
+
+        # count how many spans were marked as new
+        new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+        expect(new_tests_count).to eq(0)
 
         failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-        expect(failed_spans).to have(2).items
+        expect(failed_spans).to have(4).items
         expect(passed_spans).to have(1).items
 
         expect(test_suite_spans).to have(1).item
@@ -548,23 +562,23 @@
       end
     end
 
-    context "when total limit of failed tests to retry is zero" do
-      before do
-        skip("cucumber-ruby earlier than 9.0.0 does not support total test retries limit") unless cucumber_9_or_above
-      end
-
-      let(:total_test_retries_limit) { 0 }
+    context "when total limit of failed tests to retry is 1" do
+      let(:total_test_retries_limit) { 1 }
       let(:expected_test_run_code) { 2 }
 
       it "does not retry the test" do
-        # 1 initial run of flaky test + 1 passing = 2 spans
-        expect(test_spans).to have(2).items
+        # 1 initial run of flaky test + 4 retries + 1 passing + 1 failed run of flaky test = 7 spans
+        expect(test_spans).to have(7).items
         retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
-        expect(retries_count).to eq(0)
+        expect(retries_count).to eq(4)
+
+        # count how many spans were marked as new
+        new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+        expect(new_tests_count).to eq(0)
 
         failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
-        expect(failed_spans).to have(1).items
-        expect(passed_spans).to have(1).items
+        expect(failed_spans).to have(5).items
+        expect(passed_spans).to have(2).items
 
         expect(test_suite_spans).to have(1).item
         expect(test_suite_spans.first).to have_fail_status