diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 3c45ea19..7c6416d3 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -10,6 +10,7 @@ require_relative "../test_optimisation/coverage/transport" require_relative "../test_optimisation/coverage/writer" require_relative "../test_retries/component" +require_relative "../test_retries/null_component" require_relative "../test_visibility/component" require_relative "../test_visibility/flush" require_relative "../test_visibility/null_component" @@ -35,7 +36,7 @@ def initialize(settings) @test_visibility = TestVisibility::NullComponent.new @git_tree_upload_worker = DummyWorker.new @ci_remote = nil - @test_retries = nil + @test_retries = TestRetries::NullComponent.new # Activate CI mode if enabled if settings.ci.enabled diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb index 0f5a16fd..26a2197d 100644 --- a/lib/datadog/ci/contrib/rspec/example.rb +++ b/lib/datadog/ci/contrib/rspec/example.rb @@ -34,50 +34,71 @@ def run(*args) if ci_queue? suite_name = "#{suite_name} (ci-queue running example [#{test_name}])" - test_suite_span = test_visibility_component.start_test_suite(suite_name) + ci_queue_test_span = test_visibility_component.start_test_suite(suite_name) end - test_visibility_component.trace_test( - test_name, - suite_name, - tags: { - CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, - CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s, - CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]), - CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s, - CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters( - metadata: {"scoped_id" => metadata[:scoped_id]} - ) - }, - service: datadog_configuration[:service_name] - ) do |test_span| - test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION] - - metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr? - - result = super - - case execution_result.status - when :passed - test_span&.passed! - test_suite_span&.passed! - when :failed - test_span&.failed!(exception: execution_result.exception) - test_suite_span&.failed! - else - # :pending or nil - test_span&.skipped!( - reason: execution_result.pending_message, - exception: execution_result.pending_exception - ) - - test_suite_span&.skipped! + # don't report test to RSpec::Core::Reporter until retries are done + @skip_reporting = true + + test_retries_component.with_retries do |retry_callback| + test_visibility_component.trace_test( + test_name, + suite_name, + tags: { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s, + CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]), + CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s, + CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters( + metadata: {"scoped_id" => metadata[:scoped_id]} + ) + }, + service: datadog_configuration[:service_name] + ) do |test_span| + test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION] + + metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr? + + # before each run remove any previous exception + @exception = nil + + super + + case execution_result.status + when :passed + test_span&.passed! + when :failed + test_span&.failed!(exception: execution_result.exception) + else + # :pending or nil + test_span&.skipped!( + reason: execution_result.pending_message, + exception: execution_result.pending_exception + ) + end + + retry_callback.call(test_span) end + end - test_suite_span&.finish + # after retries are done, we can report the test to RSpec + @skip_reporting = false - result - end + # this is a special case for ci-queue, we need to finish the test suite span + ci_queue_test_span&.finish + + # Finish spec with latest retry's result + # TODO: when implementing new test retries make sure to clean @exception before calling this method + # if test passed at least once + finish(reporter) + end + + def finish(reporter) + # by default finish test but do not report it to RSpec::Core::Reporter + # it is going to be reported once after retries are done + return super unless @skip_reporting + + super(::RSpec::Core::NullReporter) end private @@ -103,6 +124,10 @@ def test_visibility_component Datadog.send(:components).test_visibility end + def test_retries_component + Datadog.send(:components).test_retries + end + def ci_queue? !!defined?(::RSpec::Queue::ExampleExtension) && self.class.ancestors.include?(::RSpec::Queue::ExampleExtension) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index b04a3938..27c74428 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require_relative "strategy/no_retry" +require_relative "strategy/retry_failed" + module Datadog module CI module TestRetries @@ -7,7 +10,8 @@ module TestRetries # - retrying failed tests - improve success rate of CI pipelines # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged class Component - attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit + attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, + :retry_failed_tests_total_limit, :retry_failed_tests_count def initialize( retry_failed_tests_max_attempts:, @@ -17,11 +21,55 @@ def initialize( @retry_failed_tests_enabled = false @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts @retry_failed_tests_total_limit = retry_failed_tests_total_limit + # counter that store the current number of failed tests retried + @retry_failed_tests_count = 0 + + @mutex = Mutex.new end def configure(library_settings) @retry_failed_tests_enabled = library_settings.flaky_test_retries_enabled? end + + def with_retries(&block) + # @type var retry_strategy: Strategy::Base + retry_strategy = nil + + test_finished_callback = lambda do |test_span| + if retry_strategy.nil? + # we always run test at least once and after first pass create a correct retry strategy + retry_strategy = build_strategy(test_span) + else + # after each retry we record the result, strategy will decide if we should retry again + retry_strategy&.record_retry(test_span) + end + end + + loop do + yield test_finished_callback + + break unless retry_strategy&.should_retry? + end + end + + def build_strategy(test_span) + @mutex.synchronize do + if should_retry_failed_test?(test_span) + Datadog.logger.debug("Failed test retry starts") + @retry_failed_tests_count += 1 + + Strategy::RetryFailed.new(max_attempts: @retry_failed_tests_max_attempts) + else + Strategy::NoRetry.new + end + end + end + + private + + def should_retry_failed_test?(test_span) + @retry_failed_tests_enabled && !!test_span&.failed? && @retry_failed_tests_count < @retry_failed_tests_total_limit + end end end end diff --git a/lib/datadog/ci/test_retries/null_component.rb b/lib/datadog/ci/test_retries/null_component.rb new file mode 100644 index 00000000..fb2f7140 --- /dev/null +++ b/lib/datadog/ci/test_retries/null_component.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require_relative "component" + +module Datadog + module CI + module TestRetries + class NullComponent < Component + attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit + + def initialize + # enabled only by remote settings + @retry_failed_tests_enabled = false + @retry_failed_tests_max_attempts = 0 + @retry_failed_tests_total_limit = 0 + end + + def configure(library_settings) + end + + def with_retries(&block) + no_action = proc {} + yield no_action + end + end + end + end +end diff --git a/lib/datadog/ci/test_retries/strategy/base.rb b/lib/datadog/ci/test_retries/strategy/base.rb new file mode 100644 index 00000000..3c69fd84 --- /dev/null +++ b/lib/datadog/ci/test_retries/strategy/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Datadog + module CI + module TestRetries + module Strategy + class Base + def should_retry? + false + end + + def record_retry(test_span) + test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true") + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_retries/strategy/no_retry.rb b/lib/datadog/ci/test_retries/strategy/no_retry.rb new file mode 100644 index 00000000..1a4541d8 --- /dev/null +++ b/lib/datadog/ci/test_retries/strategy/no_retry.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative "base" + +module Datadog + module CI + module TestRetries + module Strategy + class NoRetry < Base + def record_retry(test_span) + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_retries/strategy/retry_failed.rb b/lib/datadog/ci/test_retries/strategy/retry_failed.rb new file mode 100644 index 00000000..e5fa7fa4 --- /dev/null +++ b/lib/datadog/ci/test_retries/strategy/retry_failed.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require_relative "base" + +require_relative "../../ext/test" + +module Datadog + module CI + module TestRetries + module Strategy + class RetryFailed < Base + attr_reader :max_attempts + + def initialize(max_attempts:) + @max_attempts = max_attempts + + @attempts = 0 + @passed_once = false + end + + def should_retry? + @attempts < @max_attempts && !@passed_once + end + + def record_retry(test_span) + super + + @attempts += 1 + @passed_once = true if test_span&.passed? + + Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" } + end + end + end + end + end +end diff --git a/sig/datadog/ci/contrib/rspec/example.rbs b/sig/datadog/ci/contrib/rspec/example.rbs index 209e019c..a374c150 100644 --- a/sig/datadog/ci/contrib/rspec/example.rbs +++ b/sig/datadog/ci/contrib/rspec/example.rbs @@ -4,8 +4,8 @@ module Datadog module RSpec module Example def self.included: (untyped base) -> untyped - module InstanceMethods - include ::RSpec::Core::Example + module InstanceMethods : ::RSpec::Core::Example + @skip_reporting: bool def run: (untyped example_group_instance, untyped reporter) -> untyped @@ -14,6 +14,7 @@ module Datadog def fetch_top_level_example_group: () -> Hash[Symbol, untyped] def datadog_configuration: () -> untyped def test_visibility_component: () -> Datadog::CI::TestVisibility::Component + def test_retries_component: () -> Datadog::CI::TestRetries::Component def ci_queue?: () -> bool end end diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 9d902177..29760aee 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -8,9 +8,21 @@ module Datadog attr_reader retry_failed_tests_total_limit: Integer + attr_reader retry_failed_tests_count: Integer + + @mutex: Thread::Mutex + def initialize: (retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void + + def with_retries: () { (untyped) -> void } -> void + + def build_strategy: (Datadog::CI::Test test) -> Datadog::CI::TestRetries::Strategy::Base + + private + + def should_retry_failed_test?: (Datadog::CI::Test test) -> bool end end end diff --git a/sig/datadog/ci/test_retries/null_component.rbs b/sig/datadog/ci/test_retries/null_component.rbs new file mode 100644 index 00000000..23a1d0bc --- /dev/null +++ b/sig/datadog/ci/test_retries/null_component.rbs @@ -0,0 +1,25 @@ +module Datadog + module CI + module TestRetries + class NullComponent < Component + @retry_failed_tests_enabled: untyped + + @retry_failed_tests_max_attempts: untyped + + @retry_failed_tests_total_limit: untyped + + attr_reader retry_failed_tests_enabled: untyped + + attr_reader retry_failed_tests_max_attempts: untyped + + attr_reader retry_failed_tests_total_limit: untyped + + def initialize: () -> void + + def configure: (untyped library_settings) -> nil + + def with_retries: () { (untyped) -> untyped } -> untyped + end + end + end +end diff --git a/sig/datadog/ci/test_retries/strategy/base.rbs b/sig/datadog/ci/test_retries/strategy/base.rbs new file mode 100644 index 00000000..8440b3e3 --- /dev/null +++ b/sig/datadog/ci/test_retries/strategy/base.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module TestRetries + module Strategy + class Base + def should_retry?: () -> bool + + def record_retry: (Datadog::CI::Test test_span) -> void + end + end + end + end +end diff --git a/sig/datadog/ci/test_retries/strategy/no_retry.rbs b/sig/datadog/ci/test_retries/strategy/no_retry.rbs new file mode 100644 index 00000000..25ae4c6b --- /dev/null +++ b/sig/datadog/ci/test_retries/strategy/no_retry.rbs @@ -0,0 +1,11 @@ +module Datadog + module CI + module TestRetries + module Strategy + class NoRetry < Base + def record_retry: (Datadog::CI::Test test_span) -> void + end + end + end + end +end diff --git a/sig/datadog/ci/test_retries/strategy/retry_failed.rbs b/sig/datadog/ci/test_retries/strategy/retry_failed.rbs new file mode 100644 index 00000000..057f121e --- /dev/null +++ b/sig/datadog/ci/test_retries/strategy/retry_failed.rbs @@ -0,0 +1,21 @@ +module Datadog + module CI + module TestRetries + module Strategy + class RetryFailed < Base + attr_reader max_attempts: Integer + + @attempts: Integer + + @passed_once: bool + + def initialize: (max_attempts: Integer) -> void + + def should_retry?: () -> bool + + def record_retry: (Datadog::CI::Test test_span) -> void + end + end + end + end +end diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb index d16ee1dd..3131c505 100644 --- a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb @@ -13,6 +13,7 @@ include_context "CI mode activated" do let(:integration_name) { :rspec } + let(:flaky_test_retries_enabled) { true } end let(:run_id) { SecureRandom.random_number(2**64 - 1) } @@ -20,7 +21,7 @@ RSpec::Core::ConfigurationOptions.new([ "-Ispec/datadog/ci/contrib/ci_queue_rspec/suite_under_test", "--queue", - "list:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A1%5D:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A2%5D", + "list:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A1%5D:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A2%5D:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A3%5D", "--require", "some_test_rspec.rb", "--build", @@ -71,23 +72,24 @@ def with_new_rspec_environment expect([test_session_span, test_module_span]).to all have_fail_status # test suite spans are created for each test as for parallel execution - expect(test_suite_spans).to have(2).items + expect(test_suite_spans).to have(3).items expect(test_suite_spans).to have_tag_values_no_order( :status, - [Datadog::CI::Ext::Test::Status::FAIL, Datadog::CI::Ext::Test::Status::PASS] + [Datadog::CI::Ext::Test::Status::FAIL, Datadog::CI::Ext::Test::Status::PASS, Datadog::CI::Ext::Test::Status::SKIP] ) expect(test_suite_spans).to have_tag_values_no_order( :suite, [ "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested fails])", - "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested foo])" + "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested foo])", + "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested is skipped])" ] ) - # there is test span for every test case - expect(test_spans).to have(2).items + # there is test span for every test case + 5 retries + expect(test_spans).to have(8).items # each test span has its own test suite - expect(test_spans).to have_unique_tag_values_count(:test_suite_id, 2) + expect(test_spans).to have_unique_tag_values_count(:test_suite_id, 3) # every test span is connected to test module and test session expect(test_spans).to all have_test_tag(:test_module_id) diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb index 7b775cb9..97e4a95a 100644 --- a/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb +++ b/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb @@ -9,5 +9,8 @@ it "fails" do expect(1).to eq(2) end + + it "is skipped", skip: true do + end end end diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index fc301ce8..0f472772 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -31,6 +31,7 @@ def rspec_session_run( with_failed_test: false, with_shared_test: false, with_shared_context: false, + with_flaky_test: false, unskippable: { test: false, context: false, @@ -41,9 +42,17 @@ def rspec_session_run( test_meta = unskippable[:test] ? {Datadog::CI::Ext::Test::ITR_UNSKIPPABLE_OPTION => true} : {} context_meta = unskippable[:context] ? {Datadog::CI::Ext::Test::ITR_UNSKIPPABLE_OPTION => true} : {} suite_meta = unskippable[:suite] ? {Datadog::CI::Ext::Test::ITR_UNSKIPPABLE_OPTION => true} : {} + + max_flaky_test_failures = 4 + flaky_test_failures = 0 + + current_let_value = 0 + with_new_rspec_environment do spec = RSpec.describe "SomeTest", suite_meta do context "nested", context_meta do + let(:let_value) { current_let_value += 1 } + it "foo", test_meta do expect(1 + 1).to eq(2) end @@ -64,6 +73,18 @@ def rspec_session_run( require_relative "some_shared_context" include_context "Shared context" end + + if with_flaky_test + it "flaky" do + Datadog::CI.active_test&.set_tag("let_value", let_value) + if flaky_test_failures < max_flaky_test_failures + flaky_test_failures += 1 + expect(1 + 1).to eq(3) + else + expect(1 + 1).to eq(2) + end + end + end end end @@ -117,7 +138,7 @@ def rspec_session_run( :source_file, "spec/datadog/ci/contrib/rspec/instrumentation_spec.rb" ) - expect(first_test_span).to have_test_tag(:source_start, "90") + expect(first_test_span).to have_test_tag(:source_start, "111") expect(first_test_span).to have_test_tag( :codeowners, "[\"@DataDog/ruby-guild\", \"@DataDog/ci-app-libraries\"]" @@ -800,4 +821,115 @@ def rspec_skipped_session_run expect(test_spans).to be_empty end end + + context "session with flaky spec and failed test retries enabled" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec"} } + + let(:flaky_test_retries_enabled) { true } + end + + it "retries test until it passes" do + rspec_session_run(with_flaky_test: true) + + # 1 initial run of flaky test + 4 retries until pass + 1 passing test = 6 spans + expect(test_spans).to have(6).items + + failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } + expect(failed_spans).to have(4).items # see steps.rb + expect(passed_spans).to have(2).items + + test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } + expect(test_spans_by_test_name["nested flaky"]).to have(5).items + + # check that let values are cleared between retries + let_values = test_spans_by_test_name["nested flaky"].map { |span| span.get_tag("let_value") } + expect(let_values).to eq([1, 2, 3, 4, 5]) + + # count how many spans were marked as retries + retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } + expect(retries_count).to eq(4) + + expect(test_spans_by_test_name["nested foo"]).to have(1).item + + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_pass_status + + expect(test_session_span).to have_pass_status + end + end + + context "session with flaky spec and failed test retries enabled with insufficient retries limit" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec"} } + + let(:flaky_test_retries_enabled) { true } + let(:retry_failed_tests_max_attempts) { 3 } + end + + it "retries test until it passes" do + rspec_session_run(with_flaky_test: true) + + # 1 initial run of flaky test + 3 unsuccessful retries + 1 passing test = 5 spans + expect(test_spans).to have(5).items + + failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } + expect(failed_spans).to have(4).items + expect(passed_spans).to have(1).items + + test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } + expect(test_spans_by_test_name["nested flaky"]).to have(4).items + + # count how many spans were marked as retries + retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } + expect(retries_count).to eq(3) + + expect(test_spans_by_test_name["nested foo"]).to have(1).item + + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_fail_status + + expect(test_session_span).to have_fail_status + end + end + + context "session with flaky and failed specs and failed test retries enabled with low overall retries limit" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + let(:integration_options) { {service_name: "lspec"} } + + let(:flaky_test_retries_enabled) { true } + let(:retry_failed_tests_total_limit) { 1 } + end + + it "retries failed test with no success and bails out of retrying flaky test" do + rspec_session_run(with_flaky_test: true, with_failed_test: true) + + # 1 passing test + 1 failed test + 5 unsuccessful retries + 1 failed run of flaky test without retries = 8 spans + expect(test_spans).to have(8).items + + failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" } + expect(failed_spans).to have(7).items + expect(passed_spans).to have(1).items + + test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") } + + # it bailed out of retrying flaky test because global failed tests limit was exhausted already + expect(test_spans_by_test_name["nested flaky"]).to have(1).item + + # count how many spans were marked as retries + retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" } + expect(retries_count).to eq(5) + + # it retried failing test 5 times + expect(test_spans_by_test_name["nested fails"]).to have(6).items + + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_fail_status + + expect(test_session_span).to have_fail_status + end + end end diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index 84ce74bd..933e4e95 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -51,4 +51,113 @@ it { is_expected.to eq(retry_failed_tests_total_limit) } end + + describe "#build_strategy" do + subject { component.build_strategy(test_span) } + + let(:test_failed) { false } + let(:test_span) { instance_double(Datadog::CI::Test, failed?: test_failed) } + + before do + component.configure(library_settings) + end + + context "when retry failed tests is enabled" do + let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: true) } + + context "when test span is failed" do + let(:test_failed) { true } + + context "when failed tests retry limit is not reached" do + let(:retry_failed_tests_total_limit) { 1 } + + it "creates RetryFailed strategy" do + expect(subject).to be_a(Datadog::CI::TestRetries::Strategy::RetryFailed) + expect(subject.max_attempts).to eq(retry_failed_tests_max_attempts) + end + end + + context "when failed tests retry limit is reached" do + let(:retry_failed_tests_total_limit) { 1 } + + before do + component.build_strategy(test_span) + end + + it { is_expected.to be_a(Datadog::CI::TestRetries::Strategy::NoRetry) } + end + + context "when failed tests retry limit is reached with multithreading test runner" do + let(:threads_count) { 10 } + let(:retry_failed_tests_total_limit) { threads_count } + + before do + threads = (1..threads_count).map do + Thread.new { component.build_strategy(test_span) } + end + + threads.each(&:join) + end + + it "correctly exhausts failed tests limit" do + is_expected.to be_a(Datadog::CI::TestRetries::Strategy::NoRetry) + end + end + end + + context "when test span is passed" do + let(:test_failed) { false } + + it { is_expected.to be_a(Datadog::CI::TestRetries::Strategy::NoRetry) } + end + end + + context "when retry failed tests is disabled" do + let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: false) } + + it { is_expected.to be_a(Datadog::CI::TestRetries::Strategy::NoRetry) } + end + end + + describe "#with_retries" do + let(:test_failed) { false } + let(:test_span) { instance_double(Datadog::CI::Test, failed?: test_failed, passed?: false, set_tag: true) } + + subject(:runs_count) do + runs_count = 0 + component.with_retries do |test_finished_callback| + runs_count += 1 + test_finished_callback.call(test_span) + end + + runs_count + end + + before do + component.configure(library_settings) + end + + context "when no retries strategy is used" do + let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: false) } + + it { is_expected.to eq(1) } + end + + context "when retried failed tests strategy is used" do + let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: true) } + + context "when test span is failed" do + let(:test_failed) { true } + let(:retry_failed_tests_max_attempts) { 4 } + + it { is_expected.to eq(retry_failed_tests_max_attempts + 1) } + end + + context "when test span is passed" do + let(:test_failed) { false } + + it { is_expected.to eq(1) } + end + end + end end diff --git a/spec/datadog/ci/test_retries/strategy/retry_failed_spec.rb b/spec/datadog/ci/test_retries/strategy/retry_failed_spec.rb new file mode 100644 index 00000000..9a8d072e --- /dev/null +++ b/spec/datadog/ci/test_retries/strategy/retry_failed_spec.rb @@ -0,0 +1,30 @@ +require_relative "../../../../../lib/datadog/ci/test_retries/strategy/retry_failed" + +RSpec.describe Datadog::CI::TestRetries::Strategy::RetryFailed do + let(:max_attempts) { 3 } + subject(:strategy) { described_class.new(max_attempts: max_attempts) } + + describe "#should_retry?" do + subject { strategy.should_retry? } + + context "when the test has not passed yet" do + let(:test_span) { double(:test_span, set_tag: true, passed?: false) } + + it { is_expected.to be true } + + context "when the max attempts have been reached" do + before { max_attempts.times { strategy.record_retry(test_span) } } + + it { is_expected.to be false } + end + end + + context "when the test has passed" do + let(:test_span) { double(:test_span, set_tag: true, passed?: true) } + + before { strategy.record_retry(test_span) } + + it { is_expected.to be false } + end + end +end diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs index d08273cc..c8d6c587 100644 --- a/vendor/rbs/rspec/0/rspec.rbs +++ b/vendor/rbs/rspec/0/rspec.rbs @@ -15,11 +15,16 @@ module RSpec::Queue::ExampleExtension end module RSpec::Core::Example + @exception: untyped + + attr_reader reporter: untyped + def run: () -> untyped def execution_result: () -> untyped def metadata: () -> untyped def description: () -> String def full_description: () -> String + def finish: (untyped) -> void end module RSpec::Core::Runner