[SDTEST-173] early flake detection support for rspec and minitest

.github/workflows/add-milestone-to-pull-requests.yml

@@ -5,6 +5,9 @@ on:
 
 jobs:
   add_milestone:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     if: github.event.pull_request.merged == true && github.event.pull_request.milestone == null
     steps:

lib/datadog/ci.rb

@@ -27,7 +27,8 @@ class << self
       # ```
       # Datadog::CI.start_test_session(
       #   service: "my-web-site-tests",
-      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
+      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" },
+      #   total_tests_count: 100
       # )
       #
       # # Somewhere else after test run has ended
@@ -38,15 +39,16 @@ class << self
       #
       # @param [String] service the service name for this session (optional, defaults to DD_SERVICE or repository name)
       # @param [Hash<String,String>] tags extra tags which should be added to the test session.
+      # @param [Integer] total_tests_count the total number of tests in the test session (optional, defaults to 0) - it is used to limit the number of new tests retried within session if early flake detection is enabled
       # @return [Datadog::CI::TestSession] the active, running {Datadog::CI::TestSession}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
-      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {})
+      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {}, total_tests_count: 0)
         Utils::Telemetry.inc(
           Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
           1,
           {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::SESSION}
         )
-        test_visibility.start_test_session(service: service, tags: tags)
+        test_visibility.start_test_session(service: service, tags: tags, total_tests_count: total_tests_count)
       end
 
       # The active, unfinished test session.

lib/datadog/ci/contrib/minitest/runner.rb

@@ -8,6 +8,8 @@ module CI
     module Contrib
       module Minitest
         module Runner
+          DD_ESTIMATED_TESTS_PER_SUITE = 5
+
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
           end
@@ -18,12 +20,15 @@ def init_plugins(*args)
 
               return unless datadog_configuration[:enabled]
 
+              # minitest does not store the total number of tests, so we can't pass it to the test session
+              # instead, we use the number of test suites * DD_ESTIMATED_TESTS_PER_SUITE as a rough estimate
               test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
-                service: datadog_configuration[:service_name]
+                service: datadog_configuration[:service_name],
+                total_tests_count: (DD_ESTIMATED_TESTS_PER_SUITE * ::Minitest::Runnable.runnables.size).to_i
               )
               test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end

lib/datadog/ci/contrib/minitest/test.rb

@@ -51,6 +51,10 @@ def after_teardown
               return super unless test_span
 
               finish_with_result(test_span, result_code)
+
+              # remove failures if test passed at least once on retries
+              self.failures = [] if test_span.any_retry_passed?
+
               if Helpers.parallel?(self.class)
                 finish_with_result(test_span.test_suite, result_code)
               end

lib/datadog/ci/contrib/rspec/example.rb

@@ -64,16 +64,18 @@ def run(*args)
 
                   result = super
 
-                  # In case when test job is canceled and RSpec is quitting we don't want to report the last test
+                  # When test job is canceled and RSpec is quitting we don't want to report the last test
                   # before RSpec context unwinds. This test might have some unrelated errors that we don't want to
-                  # report.
+                  # see in Datadog.
                   return result if ::RSpec.world.wants_to_quit
 
                   case execution_result.status
                   when :passed
                     test_span&.passed!
                   when :failed
                     test_span&.failed!(exception: execution_result.exception)
+                    # if any of the retries passed, we don't fail the test run
+                    @exception = nil if test_span&.any_retry_passed?
                   else
                     # :pending or nil
                     test_span&.skipped!(
@@ -84,21 +86,20 @@ def run(*args)
                 end
               end
 
-              # after retries are done, we can report the test to RSpec
-              @skip_reporting = false
-
               # this is a special case for ci-queue, we need to finish the test suite span
               ci_queue_test_span&.finish
 
-              # Finish spec with latest retry's result
-              # TODO: when implementing new test retries make sure to clean @exception before calling this method
-              # if test passed at least once
+              # after retries are done, we can finally report the test to RSpec
+              @skip_reporting = false
               finish(reporter)
             end
 
             def finish(reporter)
-              # by default finish test but do not report it to RSpec::Core::Reporter
-              # it is going to be reported once after retries are done
+              # By default finish test but do not report it to RSpec::Core::Reporter
+              # it is going to be reported once after retries are done.
+              #
+              # We need to do this because RSpec breaks when we try to report the same example multiple times with different
+              # results.
               return super unless @skip_reporting
 
               super(::RSpec::Core::NullReporter)

lib/datadog/ci/contrib/rspec/runner.rb

@@ -23,7 +23,8 @@ def run_specs(*args)
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
-                service: datadog_configuration[:service_name]
+                service: datadog_configuration[:service_name],
+                total_tests_count: ::RSpec.world.example_count
               )
 
               test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)

lib/datadog/ci/ext/telemetry.rb

@@ -69,6 +69,7 @@ module Telemetry
         TAG_BROWSER_DRIVER = "browser_driver"
         TAG_IS_RUM = "is_rum"
         TAG_IS_RETRY = "is_retry"
+        TAG_IS_NEW = "is_new"
         TAG_LIBRARY = "library"
         TAG_ENDPOINT = "endpoint"
         TAG_ERROR_TYPE = "error_type"
@@ -80,6 +81,7 @@ module Telemetry
         TAG_COVERAGE_ENABLED = "coverage_enabled"
         TAG_ITR_SKIP_ENABLED = "itrskip_enabled"
         TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled"
+        TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
         TAG_PROVIDER = "provider"
         TAG_AUTO_INJECTED = "auto_injected"
 

lib/datadog/ci/ext/test.rb

@@ -58,8 +58,11 @@ module Test
         # version of the browser, if multiple browsers or multiple versions then this tag is empty
         TAG_BROWSER_VERSION = "test.browser.version"
 
-        # Tags for test retries
+        # Tags for retries
         TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
+        TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
+        TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
+        TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
 
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
@@ -73,6 +76,8 @@ module Test
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
         ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 
+        EARLY_FLAKE_FAULTY = "faulty"
+
         # test status as recognized by Datadog
         module Status
           PASS = "pass"

lib/datadog/ci/test.rb

@@ -145,11 +145,18 @@ def parameters
         get_tag(Ext::Test::TAG_PARAMETERS)
       end
 
+      # @internal
+      def any_retry_passed?
+        !!test_suite&.any_test_retry_passed?(test_id)
+      end
+
       private
 
-      def record_test_result(datadog_status)
-        test_id = Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
+      def test_id
+        @test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
+      end
 
+      def record_test_result(datadog_status)
         # if this test was already executed in this test suite, mark it as retried
         if test_suite&.test_executed?(test_id)
           set_tag(Ext::Test::TAG_IS_RETRY, "true")