Skip to content

Commit

Permalink
confirm that new test retries work with RSpec
Browse files Browse the repository at this point in the history
  • Loading branch information
anmarchenko committed Sep 6, 2024
1 parent ec3c099 commit 26cf239
Showing 1 changed file with 58 additions and 0 deletions.
58 changes: 58 additions & 0 deletions spec/datadog/ci/contrib/rspec/instrumentation_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -958,4 +958,62 @@ def rspec_skipped_session_run
end
end
end

context "session with early flake detection enabled" do
include_context "CI mode activated" do
let(:integration_name) { :rspec }

let(:early_flake_detection_enabled) { true }
let(:unique_tests_set) { Set.new(["SomeTest at ./spec/datadog/ci/contrib/rspec/instrumentation_spec.rb.nested foo."]) }
end

it "retries the new test 10 times" do
rspec_session_run(with_failed_test: true)

# 1 passing test + 10 new test retries + 1 failed test run = 12 spans
expect(test_spans).to have(12).items

failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
expect(failed_spans).to have(1).items
expect(passed_spans).to have(11).items

test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }

# it retried the new test 10 times
expect(test_spans_by_test_name["nested foo"]).to have(11).item

# count how many spans were marked as retries
retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
expect(retries_count).to eq(10)

expect(test_suite_spans).to have(1).item
expect(test_suite_spans.first).to have_fail_status

expect(test_session_span).to have_fail_status
end

context "when test is slower than 5 seconds" do
before do
allow_any_instance_of(Datadog::Tracing::SpanOperation).to receive(:duration).and_return(6.0)
end

it "retries the new test 5 times" do
rspec_session_run(with_failed_test: true)

# 1 passing test + 5 new test retries + 1 failed test run = 12 spans
expect(test_spans).to have(7).items

test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
# it retried the new test 5 times
expect(test_spans_by_test_name["nested foo"]).to have(6).item

# count how many spans were marked as retries
retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
expect(retries_count).to eq(5)

expect(test_suite_spans).to have(1).item
expect(test_session_span).to have_fail_status
end
end
end
end

0 comments on commit 26cf239

Please sign in to comment.