add session level tags for EFD

lib/datadog/ci/ext/test.rb

@@ -58,9 +58,11 @@ module Test
         # version of the browser, if multiple browsers or multiple versions then this tag is empty
         TAG_BROWSER_VERSION = "test.browser.version"
 
-        # Tags for test retries
+        # Tags for retries
         TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
         TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
+        TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
+        TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
 
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
@@ -74,6 +76,8 @@ module Test
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
         ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 
+        EARLY_FLAKE_FAULTY = "faulty"
+
         # test status as recognized by Datadog
         module Status
           PASS = "pass"

lib/datadog/ci/test_retries/component.rb

@@ -19,8 +19,7 @@ class Component
         attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts,
           :retry_failed_tests_total_limit, :retry_failed_tests_count,
           :retry_new_tests_enabled, :retry_new_tests_duration_thresholds,
-          :retry_new_tests_percentage_limit,
-          :retry_new_tests_unique_tests_set, :retry_new_tests_fault_reason
+          :retry_new_tests_percentage_limit, :retry_new_tests_unique_tests_set
 
         def initialize(
           retry_failed_tests_enabled:,
@@ -39,9 +38,6 @@ def initialize(
           @retry_new_tests_duration_thresholds = nil
           @retry_new_tests_percentage_limit = 0
           @retry_new_tests_unique_tests_set = Set.new
-          # indicates that retrying new tests failed and was disabled
-          @retry_new_tests_fault_reason = nil
-
           @unique_tests_client = unique_tests_client
 
           @mutex = Mutex.new
@@ -53,14 +49,18 @@ def configure(library_settings, test_session)
 
           return unless @retry_new_tests_enabled
 
+          # mark early flake detection enabled for test session
+          test_session.set_tag(Ext::Test::TAG_EARLY_FLAKE_ENABLED, "true")
+
           # configure retrying new tests
           @retry_new_tests_duration_thresholds = library_settings.slow_test_retries
           @retry_new_tests_percentage_limit = library_settings.faulty_session_threshold
           @retry_new_tests_unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
 
           if @retry_new_tests_unique_tests_set.empty?
             @retry_new_tests_enabled = false
-            @retry_new_tests_fault_reason = "unique tests set is empty"
+
+            test_session.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
 
             Datadog.logger.debug("Unique tests set is empty, retrying new tests disabled")
           else

sig/datadog/ci/ext/test.rbs

@@ -90,6 +90,12 @@ module Datadog
 
         TAG_IS_NEW: "test.is_new"
 
+        TAG_EARLY_FLAKE_ENABLED: "test.early_flake.enabled"
+
+        TAG_EARLY_FLAKE_ABORT_REASON: "test.early_flake.abort_reason"
+
+        EARLY_FLAKE_FAULTY: "faulty"
+
         module Status
           PASS: "pass"
 

sig/datadog/ci/test_retries/component.rbs

@@ -20,8 +20,6 @@ module Datadog
 
         attr_reader retry_new_tests_unique_tests_set: Set[String]
 
-        attr_reader retry_new_tests_fault_reason: String?
-
         @mutex: Thread::Mutex
 
         @unique_tests_client: Datadog::CI::TestRetries::UniqueTestsClient

spec/datadog/ci/contrib/rspec/instrumentation_spec.rb

@@ -994,6 +994,7 @@ def rspec_skipped_session_run
       expect(test_suite_spans.first).to have_fail_status
 
       expect(test_session_span).to have_fail_status
+      expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
     end
 
     context "when test is slower than 5 seconds" do
@@ -1004,7 +1005,7 @@ def rspec_skipped_session_run
       it "retries the new test 5 times" do
         rspec_session_run(with_failed_test: true)
 
-        # 1 passing test + 5 new test retries + 1 failed test run = 12 spans
+        # 1 passing test + 5 new test retries + 1 failed test run = 7 spans
         expect(test_spans).to have(7).items
 
         test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
@@ -1021,8 +1022,67 @@ def rspec_skipped_session_run
 
         expect(test_suite_spans).to have(1).item
         expect(test_session_span).to have_fail_status
+        expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
       end
     end
+
+    context "when test is slower than 10 minutes" do
+      before do
+        allow_any_instance_of(Datadog::Tracing::SpanOperation).to receive(:duration).and_return(601.0)
+      end
+
+      it "doesn't retry the new test" do
+        rspec_session_run(with_failed_test: true)
+
+        # 1 passing test + 1 failed test run = 2 spans
+        expect(test_spans).to have(2).items
+
+        test_spans_by_test_name = test_spans.group_by { |span| span.get_tag("test.name") }
+        # it retried the new test 0 times
+        expect(test_spans_by_test_name["nested foo"]).to have(1).item
+
+        # count how many spans were marked as retries
+        retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+        expect(retries_count).to eq(0)
+
+        # count how many tests were marked as new
+        new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+        expect(new_tests_count).to eq(1)
+
+        expect(test_suite_spans).to have(1).item
+        expect(test_session_span).to have_fail_status
+        expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
+      end
+    end
+  end
+
+  context "session with early flake detection enabled but unique tests set is empty" do
+    include_context "CI mode activated" do
+      let(:integration_name) { :rspec }
+
+      let(:early_flake_detection_enabled) { true }
+    end
+
+    it "retries the new test 10 times and the flaky test until it passes" do
+      rspec_session_run
+
+      expect(test_spans).to have(1).item
+
+      # count how many spans were marked as retries
+      retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+      expect(retries_count).to eq(0)
+
+      # count how many tests were marked as new
+      new_tests_count = test_spans.count { |span| span.get_tag("test.is_new") == "true" }
+      expect(new_tests_count).to eq(0)
+
+      expect(test_suite_spans).to have(1).item
+      expect(test_suite_spans.first).to have_pass_status
+
+      expect(test_session_span).to have_pass_status
+      expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
+      expect(test_session_span).to have_test_tag(:early_flake_abort_reason, "faulty")
+    end
   end
 
   context "session with early flake detection enabled and retrying failed tests enabled" do
@@ -1061,6 +1121,7 @@ def rspec_skipped_session_run
       expect(test_suite_spans.first).to have_pass_status
 
       expect(test_session_span).to have_pass_status
+      expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
     end
   end
 
@@ -1107,6 +1168,7 @@ def rspec_skipped_session_run
       expect(test_suite_spans.first).to have_pass_status
 
       expect(test_session_span).to have_pass_status
+      expect(test_session_span).to have_test_tag(:early_flake_enabled, "true")
     end
   end
 end