Merge pull request #167 from DataDog/anmarchenko/itr_unskippable_tests

[CIVIS-9332] unskippable tests for ITR

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -70,12 +70,15 @@ def on_test_case_started(event)
 
             start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
 
-            CI.start_test(
+            test_span = CI.start_test(
               event.test_case.name,
               test_suite_name,
               tags: tags,
               service: configuration[:service_name]
             )
+            if event.test_case.match_tags?("@#{CI::Ext::Test::ITR_UNSKIPPABLE_OPTION}")
+              test_span&.itr_unskippable!
+            end
           end
 
           def on_test_case_finished(event)

lib/datadog/ci/contrib/minitest/patcher.rb

@@ -2,7 +2,7 @@
 
 require_relative "runner"
 require_relative "reporter"
-require_relative "hooks"
+require_relative "test"
 require_relative "runnable"
 
 module Datadog
@@ -25,7 +25,7 @@ def patch
             # test suites (when not executed concurrently)
             ::Minitest::Runnable.include(Runnable)
             # tests; test suites (when executed concurrently)
-            ::Minitest::Test.include(Hooks)
+            ::Minitest::Test.include(Test)
             # test session finish
             ::Minitest::CompositeReporter.include(Reporter)
           end

lib/datadog/ci/contrib/minitest/test.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "../../git/local_repository"
+require_relative "ext"
+require_relative "helpers"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        # Lifecycle hooks to instrument Minitest::Test
+        module Test
+          def self.included(base)
+            base.prepend(InstanceMethods)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module InstanceMethods
+            def before_setup
+              super
+              return unless datadog_configuration[:enabled]
+
+              test_suite_name = Helpers.test_suite_name(self.class, name)
+              if Helpers.parallel?(self.class)
+                test_suite_name = "#{test_suite_name} (#{name} concurrently)"
+
+                # for parallel execution we need to start a new test suite for each test
+                CI.start_test_suite(test_suite_name)
+              end
+
+              source_file, line_number = method(name).source_location
+
+              test_span = CI.start_test(
+                name,
+                test_suite_name,
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
+                  CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(source_file),
+                  CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+              test_span&.itr_unskippable! if self.class.dd_suite_unskippable? || self.class.dd_test_unskippable?(name)
+              skip(CI::Ext::Test::ITR_TEST_SKIP_REASON) if test_span&.skipped_by_itr?
+            end
+
+            def after_teardown
+              test_span = CI.active_test
+              return super unless test_span
+
+              finish_with_result(test_span, result_code)
+              if Helpers.parallel?(self.class)
+                finish_with_result(test_span.test_suite, result_code)
+              end
+
+              super
+            end
+
+            private
+
+            def finish_with_result(span, result_code)
+              return unless span
+
+              case result_code
+              when "."
+                span.passed!
+              when "E", "F"
+                span.failed!(exception: failure)
+              when "S"
+                span.skipped!(reason: failure.message)
+              end
+              span.finish
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+
+          module ClassMethods
+            def datadog_itr_unskippable(*args)
+              if args.nil? || args.empty?
+                @datadog_itr_unskippable_suite = true
+              else
+                @datadog_itr_unskippable_tests = args
+              end
+            end
+
+            def dd_suite_unskippable?
+              @datadog_itr_unskippable_suite
+            end
+
+            def dd_test_unskippable?(test_name)
+              return false unless @datadog_itr_unskippable_tests
+
+              @datadog_itr_unskippable_tests.include?(test_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/rspec/example.rb

@@ -50,6 +50,8 @@ def run(*)
                 },
                 service: datadog_configuration[:service_name]
               ) do |test_span|
+                test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+
                 metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr?
 
                 result = super

lib/datadog/ci/ext/test.rb

@@ -8,7 +8,6 @@ module Ext
       module Test
         CONTEXT_ORIGIN = "ciapp-test"
 
-        TAG_ARGUMENTS = "test.arguments"
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
@@ -29,6 +28,8 @@ module Test
         TAG_ITR_TEST_SKIPPING_COUNT = "test.itr.tests_skipping.count"
         TAG_ITR_SKIPPED_BY_ITR = "test.skipped_by_itr"
         TAG_ITR_TESTS_SKIPPED = "_dd.ci.itr.tests_skipped"
+        TAG_ITR_UNSKIPPABLE = "test.itr.unskippable"
+        TAG_ITR_FORCED_RUN = "test.itr.forced_run"
 
         # Code coverage tags
         TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
@@ -58,6 +59,7 @@ module Test
         # we use test skipping for Ruby
         ITR_TEST_SKIPPING_MODE = "test"
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
+        ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 
         # test status as recognized by Datadog
         module Status

lib/datadog/ci/span.rb

@@ -102,6 +102,13 @@ def set_tag(key, value)
         tracer_span.set_tag(key, value)
       end
 
+      # Removes tag by key.
+      # @param [String] key the key of the tag.
+      # @return [void]
+      def clear_tag(key)
+        tracer_span.clear_tag(key)
+      end
+
       # Sets metric value by key.
       # @param [String] key the key of the metric.
       # @param [Numeric] value the value of the metric.

lib/datadog/ci/test.rb

@@ -69,6 +69,24 @@ def skipped_by_itr?
         get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
       end
 
+      # Marks this test as unskippable by the intelligent test runner.
+      # This must be done before the test execution starts.
+      #
+      # Examples of tests that should be unskippable:
+      # - tests that read files from disk
+      # - tests that make network requests
+      # - tests that call external processes
+      # - tests that use forking or threading
+      #
+      # @return [void]
+      def itr_unskippable!
+        set_tag(Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
+        if skipped_by_itr?
+          clear_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR)
+          set_tag(Ext::Test::TAG_ITR_FORCED_RUN, "true")
+        end
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!

sig/datadog/ci/contrib/minitest/test.rbs

@@ -0,0 +1,32 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Test
+          module ClassMethods
+            def datadog_itr_unskippable: (untyped args) -> void
+
+            def dd_suite_unskippable?: () -> bool
+
+            def dd_test_unskippable?: (String? test_name) -> bool
+          end
+
+          module InstanceMethods : ::Minitest::Test
+            include ::Minitest::Test::LifecycleHooks
+            extend ClassMethods
+
+            def before_setup: () -> (nil | untyped)
+
+            def after_teardown: () -> untyped
+
+            private
+
+            def datadog_configuration: () -> untyped
+
+            def finish_with_result: (Datadog::CI::Span? span, String result_code) -> void
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/ext/test.rbs

@@ -4,8 +4,6 @@ module Datadog
       module Test
         CONTEXT_ORIGIN: "ciapp-test"
 
-        TAG_ARGUMENTS: "test.arguments"
-
         TAG_FRAMEWORK: "test.framework"
 
         TAG_FRAMEWORK_VERSION: "test.framework_version"
@@ -42,6 +40,10 @@ module Datadog
 
         TAG_ITR_TESTS_SKIPPED: "_dd.ci.itr.tests_skipped"
 
+        TAG_ITR_UNSKIPPABLE: "test.itr.unskippable"
+
+        TAG_ITR_FORCED_RUN: "test.itr.forced_run"
+
         TAG_CODE_COVERAGE_ENABLED: "test.code_coverage.enabled"
 
         TAG_TEST_SESSION_ID: "_test.session_id"
@@ -72,6 +74,8 @@ module Datadog
 
         ITR_TEST_SKIP_REASON: String
 
+        ITR_UNSKIPPABLE_OPTION: :datadog_itr_unskippable
+
         module Status
           PASS: "pass"
 

sig/datadog/ci/span.rbs

@@ -31,6 +31,8 @@ module Datadog
 
       def set_tag: (String key, untyped? value) -> void
 
+      def clear_tag: (String key) -> void
+
       def set_metric: (String key, untyped value) -> void
 
       def set_tags: (Hash[untyped, untyped] tags) -> void

sig/datadog/ci/test.rbs

@@ -8,6 +8,7 @@ module Datadog
       def test_module_id: () -> String?
       def test_session_id: () -> String?
       def skipped_by_itr?: () -> bool
+      def itr_unskippable!: () -> void
       def source_file: () -> String?
       def parameters: () -> String?