diff --git a/lib/datadog/ci/contrib/cucumber/configuration_override.rb b/lib/datadog/ci/contrib/cucumber/configuration_override.rb deleted file mode 100644 index e3ca42b3..00000000 --- a/lib/datadog/ci/contrib/cucumber/configuration_override.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -require_relative "formatter" - -module Datadog - module CI - module Contrib - module Cucumber - # Changes behaviour of Cucumber::Configuration class - module ConfigurationOverride - def self.included(base) - base.prepend(InstanceMethods) - end - - # Instance methods for configuration - module InstanceMethods - def retry_attempts - super if !datadog_test_retries_component&.retry_failed_tests_enabled - - datadog_test_retries_component&.retry_failed_tests_max_attempts - end - - def retry_total_tests - super if !datadog_test_retries_component&.retry_failed_tests_enabled - - datadog_test_retries_component&.retry_failed_tests_total_limit - end - - def datadog_test_retries_component - Datadog.send(:components).test_retries - end - end - end - end - end - end -end diff --git a/lib/datadog/ci/contrib/cucumber/filter.rb b/lib/datadog/ci/contrib/cucumber/filter.rb new file mode 100644 index 00000000..96b11a4f --- /dev/null +++ b/lib/datadog/ci/contrib/cucumber/filter.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Contrib + module Cucumber + class Filter < ::Cucumber::Core::Filter.new(:configuration) + def test_case(test_case) + test_retries_component.reset_retries! unless test_case_seen[test_case] + + test_case_seen[test_case] = true + configuration.on_event(:test_case_finished) do |event| + next unless retry_required?(test_case, event) + + test_case.describe_to(receiver) + end + + super + end + + private + + def retry_required?(test_case, event) + return false unless event.test_case == test_case + + test_retries_component.should_retry? + end + + def test_case_seen + @test_case_seen ||= Hash.new { |h, k| h[k] = false } + end + + def test_retries_component + @test_retries_component ||= Datadog.send(:components).test_retries + end + end + end + end + end +end diff --git a/lib/datadog/ci/contrib/cucumber/instrumentation.rb b/lib/datadog/ci/contrib/cucumber/instrumentation.rb index 75d3f310..3e8fa06a 100644 --- a/lib/datadog/ci/contrib/cucumber/instrumentation.rb +++ b/lib/datadog/ci/contrib/cucumber/instrumentation.rb @@ -18,10 +18,24 @@ module InstanceMethods def formatters existing_formatters = super - @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration) + @datadog_formatter ||= Formatter.new(@configuration) [@datadog_formatter] + existing_formatters end + def filters + require_relative "filter" + + filters_list = super + datadog_filter = Filter.new(@configuration) + unless @configuration.dry_run? + # insert our filter the pre-last position because Cucumber::Filters::PrepareWorld must be the last one + # see: + # https://github.com/cucumber/cucumber-ruby/blob/58dd8f12c0ac5f4e607335ff2e7d385c1ed25899/lib/cucumber/runtime.rb#L266 + filters_list.insert(-2, datadog_filter) + end + filters_list + end + def begin_scenario(test_case) if Datadog::CI.active_test&.skipped_by_itr? raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON diff --git a/lib/datadog/ci/contrib/cucumber/patcher.rb b/lib/datadog/ci/contrib/cucumber/patcher.rb index 5a3f9d8d..0459ba18 100644 --- a/lib/datadog/ci/contrib/cucumber/patcher.rb +++ b/lib/datadog/ci/contrib/cucumber/patcher.rb @@ -3,7 +3,6 @@ require "datadog/tracing/contrib/patcher" require_relative "instrumentation" -require_relative "configuration_override" module Datadog module CI @@ -21,7 +20,6 @@ def target_version def patch ::Cucumber::Runtime.include(Instrumentation) - ::Cucumber::Configuration.include(ConfigurationOverride) end end end diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 2bbd3cb3..6bd7ac1c 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -53,15 +53,15 @@ def configure(library_settings, test_session) end def with_retries(&block) - self.current_retry_driver = nil + reset_retries! loop do yield - break unless current_retry_driver&.should_retry? + break unless should_retry? end ensure - self.current_retry_driver = nil + reset_retries! end def build_driver(test_span) @@ -89,6 +89,15 @@ def record_test_span_duration(tracer_span) current_retry_driver&.record_duration(tracer_span.duration) end + # this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method + def reset_retries! + self.current_retry_driver = nil + end + + def should_retry? + !!current_retry_driver&.should_retry? + end + private def current_retry_driver diff --git a/lib/datadog/ci/test_retries/null_component.rb b/lib/datadog/ci/test_retries/null_component.rb index fb2f7140..7da232fa 100644 --- a/lib/datadog/ci/test_retries/null_component.rb +++ b/lib/datadog/ci/test_retries/null_component.rb @@ -6,13 +6,7 @@ module Datadog module CI module TestRetries class NullComponent < Component - attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit - def initialize - # enabled only by remote settings - @retry_failed_tests_enabled = false - @retry_failed_tests_max_attempts = 0 - @retry_failed_tests_total_limit = 0 end def configure(library_settings) @@ -22,6 +16,13 @@ def with_retries(&block) no_action = proc {} yield no_action end + + def reset_retries! + end + + def should_retry? + false + end end end end diff --git a/sig/datadog/ci/contrib/cucumber/configuration_override.rbs b/sig/datadog/ci/contrib/cucumber/configuration_override.rbs deleted file mode 100644 index 043ab426..00000000 --- a/sig/datadog/ci/contrib/cucumber/configuration_override.rbs +++ /dev/null @@ -1,18 +0,0 @@ -module Datadog - module CI - module Contrib - module Cucumber - module ConfigurationOverride - def self.included: (untyped base) -> untyped - module InstanceMethods : ::Cucumber::Configuration - def retry_attempts: () -> Integer? - - def retry_total_tests: () -> Integer? - - def datadog_test_retries_component: () -> Datadog::CI::TestRetries::Component? - end - end - end - end - end -end diff --git a/sig/datadog/ci/contrib/cucumber/filter.rbs b/sig/datadog/ci/contrib/cucumber/filter.rbs new file mode 100644 index 00000000..1d3d834d --- /dev/null +++ b/sig/datadog/ci/contrib/cucumber/filter.rbs @@ -0,0 +1,23 @@ +module Datadog + module CI + module Contrib + module Cucumber + class Filter < ::Cucumber::Core::Filter + @test_case_seen: untyped + + @test_retries_component: untyped + + def test_case: (untyped test_case) -> untyped + + private + + def retry_required?: (untyped test_case, untyped event) -> (false | untyped) + + def test_case_seen: () -> untyped + + def test_retries_component: () -> untyped + end + end + end + end +end diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index bd9f975a..3654e346 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -20,6 +20,10 @@ module Datadog def record_test_span_duration: (Datadog::Tracing::SpanOperation span) -> void + def reset_retries!: () -> void + + def should_retry?: () -> bool + private def current_retry_driver: () -> Datadog::CI::TestRetries::Driver::Base? diff --git a/spec/datadog/ci/contrib/cucumber/features/flaky.feature b/spec/datadog/ci/contrib/cucumber/features/flaky.feature index 89cf18c3..b41001a2 100644 --- a/spec/datadog/ci/contrib/cucumber/features/flaky.feature +++ b/spec/datadog/ci/contrib/cucumber/features/flaky.feature @@ -3,6 +3,10 @@ Feature: When I have flaky test When flaky Then datadog + Scenario: another flaky scenario + When flaky + Then datadog + Scenario: this scenario just passes When datadog Then datadog diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb index 84fe8ff6..c4751fde 100644 --- a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb +++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb @@ -43,5 +43,7 @@ if flaky_test_executions < max_flaky_test_failures flaky_test_executions += 1 raise "Flaky test failure" + else + flaky_test_executions = 0 end end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index eb204aa0..de046303 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -3,7 +3,7 @@ require "cucumber" require "securerandom" -RSpec.describe "Cucumber formatter" do +RSpec.describe "Cucumber instrumentation" do let(:cucumber_features_root) { File.join(__dir__, "features") } let(:enable_retries) { false } let(:single_test_retries_count) { 5 } @@ -477,19 +477,24 @@ end it "retries the test several times and correctly tracks result of every invocation" do - # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans - expect(test_spans).to have(6).items + # 1 initial run of flaky test + 4 retries until pass + 1 passing test + 1 other flaky + 4 retries until pass = 11 spans + expect(test_spans).to have(11).items failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } - expect(failed_spans).to have(4).items # see steps.rb - expect(passed_spans).to have(2).items + expect(failed_spans).to have(8).items # see steps.rb + expect(passed_spans).to have(3).items test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } expect(test_spans_by_test_name["very flaky scenario"]).to have(5).items + expect(test_spans_by_test_name["another flaky scenario"]).to have(5).items # count how many spans were marked as retries retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } - expect(retries_count).to eq(4) + expect(retries_count).to eq(8) + + # count how many spans were marked as new + new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" } + expect(new_tests_count).to eq(0) expect(test_spans_by_test_name["this scenario just passes"]).to have(1).item @@ -505,19 +510,24 @@ let(:feature_file_to_run) { "flaky.feature" } it "retries the test several times and correctly tracks result of every invocation" do - # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans - expect(test_spans).to have(6).items + # 1 initial run of flaky test + 4 retries until pass + 1 passing test + 1 other flaky + 4 retries = 11 spans + expect(test_spans).to have(11).items failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } - expect(failed_spans).to have(4).items # see steps.rb - expect(passed_spans).to have(2).items + expect(failed_spans).to have(8).items # see steps.rb + expect(passed_spans).to have(3).items test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } expect(test_spans_by_test_name["very flaky scenario"]).to have(5).items + expect(test_spans_by_test_name["another flaky scenario"]).to have(5).items # count how many spans were marked as retries retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } - expect(retries_count).to eq(4) + expect(retries_count).to eq(8) + + # count how many spans were marked as new + new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" } + expect(new_tests_count).to eq(0) expect(test_spans_by_test_name["this scenario just passes"]).to have(1).item @@ -532,13 +542,17 @@ let(:expected_test_run_code) { 2 } it "retries the test once" do - # 1 initial run of flaky test + 1 retry + 1 passing = 3 spans - expect(test_spans).to have(3).items + # 1 initial run of flaky test + 1 retry + 1 passing + 1 other flaky + 1 retry = 5 spans + expect(test_spans).to have(5).items retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } - expect(retries_count).to eq(1) + expect(retries_count).to eq(2) + + # count how many spans were marked as new + new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" } + expect(new_tests_count).to eq(0) failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } - expect(failed_spans).to have(2).items + expect(failed_spans).to have(4).items expect(passed_spans).to have(1).items expect(test_suite_spans).to have(1).item @@ -548,23 +562,23 @@ end end - context "when total limit of failed tests to retry is zero" do - before do - skip("cucumber-ruby earlier than 9.0.0 does not support total test retries limit") unless cucumber_9_or_above - end - - let(:total_test_retries_limit) { 0 } + context "when total limit of failed tests to retry is 1" do + let(:total_test_retries_limit) { 1 } let(:expected_test_run_code) { 2 } it "does not retry the test" do - # 1 initial run of flaky test + 1 passing = 2 spans - expect(test_spans).to have(2).items + # 1 initial run of flaky test + 4 retries + 1 passing + 1 failed run of flaky test = 7 spans + expect(test_spans).to have(7).items retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } - expect(retries_count).to eq(0) + expect(retries_count).to eq(4) + + # count how many spans were marked as new + new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" } + expect(new_tests_count).to eq(0) failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } - expect(failed_spans).to have(1).items - expect(passed_spans).to have(1).items + expect(failed_spans).to have(5).items + expect(passed_spans).to have(2).items expect(test_suite_spans).to have(1).item expect(test_suite_spans.first).to have_fail_status diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index 6c14164c..cd9262d8 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -15,6 +15,8 @@ end class Cucumber::Configuration def retry_attempts: () -> Integer? def retry_total_tests: () -> Integer? + + def on_event: (Symbol event_name) { (untyped event) -> void } -> void end module Cucumber::Formatter @@ -55,4 +57,14 @@ end class Cucumber::Messages::Feature def name: () -> String +end + +class Cucumber::Core::Filter + def initialize: (Symbol param) -> void + + def configuration: () -> Cucumber::Configuration + + def test_case: (untyped test_case) -> void + + def receiver: () -> untyped end \ No newline at end of file