Merge pull request #262 from DataDog/anmarchenko/rspec_skip_example_g…

…roup

[SDTEST-1229] Skip before(:all) context hooks when all examples are skipped

lib/datadog/ci/contrib/rspec/example.rb

@@ -21,42 +21,25 @@ def run(*args)
               return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
-              test_name = full_description.strip
-              if metadata[:description].empty?
-                # for unnamed it blocks this appends something like "example at ./spec/some_spec.rb:10"
-                test_name << " #{description}"
-              end
-
-              test_suite_description = fetch_top_level_example_group[:description]
-              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
-
-              # remove example group description from test name to avoid duplication
-              test_name = test_name.sub(test_suite_description, "").strip
-
-              if ci_queue?
-                suite_name = "#{suite_name} (ci-queue running example [#{test_name}])"
-                ci_queue_test_span = test_visibility_component.start_test_suite(suite_name)
-              end
+              test_suite_span = test_visibility_component.start_test_suite(datadog_test_suite_name) if ci_queue?
 
               # don't report test to RSpec::Core::Reporter until retries are done
               @skip_reporting = true
 
               test_retries_component.with_retries do
                 test_visibility_component.trace_test(
-                  test_name,
-                  suite_name,
+                  datadog_test_name,
+                  datadog_test_suite_name,
                   tags: {
                     CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                     CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s,
                     CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
                     CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
-                    CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters(
-                      metadata: {"scoped_id" => metadata[:scoped_id]}
-                    )
+                    CI::Ext::Test::TAG_PARAMETERS => datadog_test_parameters
                   },
                   service: datadog_configuration[:service_name]
                 ) do |test_span|
-                  test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+                  test_span&.itr_unskippable! if datadog_unskippable?
 
                   metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr?
 
@@ -87,8 +70,8 @@ def run(*args)
                 end
               end
 
-              # this is a special case for ci-queue, we need to finish the test suite span
-              ci_queue_test_span&.finish
+              # this is a special case for ci-queue, we need to finish the test suite span created for a single test
+              test_suite_span&.finish
 
               # after retries are done, we can finally report the test to RSpec
               @skip_reporting = false
@@ -106,6 +89,18 @@ def finish(reporter)
               super(::RSpec::Core::NullReporter)
             end
 
+            def datadog_test_id
+              @datadog_test_id ||= Utils::TestRun.datadog_test_id(
+                datadog_test_name,
+                datadog_test_suite_name,
+                datadog_test_parameters
+              )
+            end
+
+            def datadog_unskippable?
+              !!metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+            end
+
             private
 
             def fetch_top_level_example_group
@@ -129,6 +124,45 @@ def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
 
+            def datadog_test_suite_description
+              @datadog_test_suite_description ||= fetch_top_level_example_group[:description]
+            end
+
+            def datadog_test_name
+              return @datadog_test_name if defined?(@datadog_test_name)
+
+              test_name = full_description.strip
+              if metadata[:description].empty?
+                # for unnamed it blocks this appends something like "example at ./spec/some_spec.rb:10"
+                test_name << " #{description}"
+              end
+
+              # remove example group description from test name to avoid duplication
+              test_name = test_name.sub(datadog_test_suite_description, "").strip
+
+              @datadog_test_name = test_name
+            end
+
+            def datadog_test_suite_name
+              return @datadog_test_suite_name if defined?(@datadog_test_suite_name)
+
+              suite_name = "#{datadog_test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              if ci_queue?
+                suite_name = "#{suite_name} (ci-queue running example [#{datadog_test_name}])"
+              end
+
+              @datadog_test_suite_name = suite_name
+            end
+
+            def datadog_test_parameters
+              return @datadog_test_parameters if defined?(@datadog_test_parameters)
+
+              @datadog_test_parameters = Utils::TestRun.test_parameters(
+                metadata: {"scoped_id" => metadata[:scoped_id]}
+              )
+            end
+
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -18,10 +18,17 @@ module ClassMethods
             def run(reporter = ::RSpec::Core::NullReporter)
               return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
+
+              # skip the context hooks if all descendant example are going to be skipped
+              # IMPORTANT: must happen before top_level? check because skipping hooks must happen
+              # even if it is a nested context
+              metadata[:skip] = true if all_examples_skipped_by_datadog?
+
+              # return early because we create Datadog::CI::TestSuite only for top-level example groups
               return super unless top_level?
 
               suite_name = "#{description} at #{file_path}"
-              test_suite = test_visibility_component.start_test_suite(
+              test_suite = test_visibility_component&.start_test_suite(
                 suite_name,
                 tags: {
                   CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
@@ -47,13 +54,23 @@ def run(reporter = ::RSpec::Core::NullReporter)
 
             private
 
+            def all_examples_skipped_by_datadog?
+              descendant_filtered_examples.all? do |example|
+                !example.datadog_unskippable? && test_optimisation_component&.skippable?(example)
+              end
+            end
+
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
 
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end
+
+            def test_optimisation_component
+              Datadog.send(:components).test_optimisation
+            end
           end
         end
       end

lib/datadog/ci/test.rb

@@ -17,6 +17,11 @@ def name
         get_tag(Ext::Test::TAG_NAME)
       end
 
+      # @return [String] the test id according to Datadog's test impact analysis.
+      def datadog_test_id
+        @datadog_test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
+      end
+
       # Finishes the current test.
       # @return [void]
       def finish
@@ -140,22 +145,18 @@ def parameters
 
       # @internal
       def any_retry_passed?
-        !!test_suite&.any_test_retry_passed?(test_id)
+        !!test_suite&.any_test_retry_passed?(datadog_test_id)
       end
 
       private
 
-      def test_id
-        @test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
-      end
-
       def record_test_result(datadog_status)
         # if this test was already executed in this test suite, mark it as retried
-        if test_suite&.test_executed?(test_id)
+        if test_suite&.test_executed?(datadog_test_id)
           set_tag(Ext::Test::TAG_IS_RETRY, "true")
         end
 
-        test_suite&.record_test_result(test_id, datadog_status)
+        test_suite&.record_test_result(datadog_test_id, datadog_status)
       end
     end
   end

lib/datadog/ci/test_optimisation/component.rb

@@ -144,21 +144,26 @@ def stop_coverage(test)
           event
         end
 
+        def skippable?(test)
+          return false if !enabled? || !skipping_tests?
+
+          @skippable_tests.include?(test.datadog_test_id)
+        end
+
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
-          datadog_test_id = Utils::TestRun.datadog_test_id(test.name, test.test_suite_name, test.parameters)
-          if @skippable_tests.include?(datadog_test_id)
+          if skippable?(test)
             if forked?
               Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
               return
             end
 
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
-            Datadog.logger.debug { "Marked test as skippable: #{datadog_test_id}" }
+            Datadog.logger.debug { "Marked test as skippable: #{test.datadog_test_id}" }
           else
-            Datadog.logger.debug { "Test is not skippable: #{datadog_test_id}" }
+            Datadog.logger.debug { "Test is not skippable: #{test.datadog_test_id}" }
           end
         end
 

sig/datadog/ci/contrib/rspec/example.rbs

@@ -7,8 +7,18 @@ module Datadog
           module InstanceMethods : ::RSpec::Core::Example
             @skip_reporting: bool
 
+            @datadog_test_suite_description: String
+
+            @datadog_test_id: String
+            @datadog_test_name: String
+            @datadog_test_suite_name: String
+            @datadog_test_parameters: String
+
             def run: (untyped example_group_instance, untyped reporter) -> untyped
 
+            def datadog_test_id: () -> String
+            def datadog_unskippable?: () -> bool
+
             private
 
             def fetch_top_level_example_group: () -> Hash[Symbol, untyped]
@@ -17,6 +27,11 @@ module Datadog
             def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
             def test_retries_component: () -> Datadog::CI::TestRetries::Component
             def ci_queue?: () -> bool
+
+            def datadog_test_suite_description: () -> String
+            def datadog_test_name: () -> String
+            def datadog_test_suite_name: () -> String
+            def datadog_test_parameters: () -> String
           end
         end
       end

sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -12,9 +12,13 @@ module Datadog
 
             private
 
+            def all_examples_skipped_by_datadog?: () -> bool
+
             def datadog_configuration: () -> untyped
 
-            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component?
+
+            def test_optimisation_component: () -> Datadog::CI::TestOptimisation::Component?
           end
         end
       end

sig/datadog/ci/test.rbs

@@ -1,8 +1,9 @@
 module Datadog
   module CI
     class Test < Span
-      @test_id: String
+      @datadog_test_id: String
 
+      def datadog_test_id: () -> String
       def finish: () -> void
       def test_suite: () -> Datadog::CI::TestSuite?
       def test_suite_id: () -> String?
@@ -17,7 +18,6 @@ module Datadog
 
       private
 
-      def test_id: () -> String
       def record_test_result: (String datadog_status) -> void
     end
   end

sig/datadog/ci/test_optimisation/component.rbs

@@ -45,6 +45,8 @@ module Datadog
 
         def stop_coverage: (Datadog::CI::Test test) -> Datadog::CI::TestOptimisation::Coverage::Event?
 
+        def skippable?: (Datadog::CI::Test test) -> bool
+
         def mark_if_skippable: (Datadog::CI::Test test) -> void
 
         def count_skipped_test: (Datadog::CI::Test test) -> void