first stab on unskippable tests in minitest

lib/datadog/ci/contrib/minitest/hooks.rb

@@ -11,64 +11,92 @@ module Contrib
       module Minitest
         # Lifecycle hooks to instrument Minitest::Test
         module Hooks
-          def before_setup
-            super
-            return unless datadog_configuration[:enabled]
+          def self.included(base)
+            base.prepend(InstanceMethods)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module InstanceMethods
+            def before_setup
+              super
+              return unless datadog_configuration[:enabled]
+
+              test_suite_name = Helpers.test_suite_name(self.class, name)
+              if Helpers.parallel?(self.class)
+                test_suite_name = "#{test_suite_name} (#{name} concurrently)"
+
+                # for parallel execution we need to start a new test suite for each test
+                CI.start_test_suite(test_suite_name)
+              end
 
-            test_suite_name = Helpers.test_suite_name(self.class, name)
-            if Helpers.parallel?(self.class)
-              test_suite_name = "#{test_suite_name} (#{name} concurrently)"
+              source_file, line_number = method(name).source_location
 
-              # for parallel execution we need to start a new test suite for each test
-              CI.start_test_suite(test_suite_name)
+              test_span = CI.start_test(
+                name,
+                test_suite_name,
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
+                  CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(source_file),
+                  CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+              test_span&.itr_unskippable! if self.class.dd_suite_unskippable? || self.class.dd_test_unskippable?(name)
+              skip(CI::Ext::Test::ITR_TEST_SKIP_REASON) if test_span&.skipped_by_itr?
             end
 
-            source_file, line_number = method(name).source_location
-
-            test_span = CI.start_test(
-              name,
-              test_suite_name,
-              tags: {
-                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(source_file),
-                CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
-              },
-              service: datadog_configuration[:service_name]
-            )
-            skip(CI::Ext::Test::ITR_TEST_SKIP_REASON) if test_span&.skipped_by_itr?
-          end
+            def after_teardown
+              test_span = CI.active_test
+              return super unless test_span
 
-          def after_teardown
-            test_span = CI.active_test
-            return super unless test_span
+              finish_with_result(test_span, result_code)
+              if Helpers.parallel?(self.class)
+                finish_with_result(test_span.test_suite, result_code)
+              end
 
-            finish_with_result(test_span, result_code)
-            if Helpers.parallel?(self.class)
-              finish_with_result(test_span.test_suite, result_code)
+              super
             end
 
-            super
-          end
+            private
 
-          private
+            def finish_with_result(span, result_code)
+              return unless span
 
-          def finish_with_result(span, result_code)
-            return unless span
+              case result_code
+              when "."
+                span.passed!
+              when "E", "F"
+                span.failed!(exception: failure)
+              when "S"
+                span.skipped!(reason: failure.message)
+              end
+              span.finish
+            end
 
-            case result_code
-            when "."
-              span.passed!
-            when "E", "F"
-              span.failed!(exception: failure)
-            when "S"
-              span.skipped!(reason: failure.message)
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
             end
-            span.finish
           end
 
-          def datadog_configuration
-            Datadog.configuration.ci[:minitest]
+          module ClassMethods
+            def datadog_itr_unskippable(*args)
+              if args.nil? || args.empty?
+                @datadog_itr_unskippable_suite = true
+              else
+                @datadog_itr_unskippable_tests = args
+              end
+            end
+
+            def dd_suite_unskippable?
+              @datadog_itr_unskippable_suite
+            end
+
+            def dd_test_unskippable?(test_name)
+              return false unless @datadog_itr_unskippable_tests
+
+              @datadog_itr_unskippable_tests.include?(test_name)
+            end
           end
         end
       end

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -775,6 +775,84 @@ def test_2
           expect(first_test_suite_span).to have_skip_status
         end
       end
+
+      context "unskippable suite" do
+        before(:context) do
+          Minitest::Runnable.reset
+
+          class UnskippableTest < Minitest::Test
+            datadog_itr_unskippable
+
+            def test_1
+            end
+
+            def test_2
+            end
+          end
+
+          class ActuallySkippableTest < Minitest::Test
+            def test_1
+            end
+          end
+        end
+
+        let(:itr_skippable_tests) do
+          Set.new(
+            [
+              "UnskippableTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb.test_1.",
+              "UnskippableTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb.test_2.",
+              "ActuallySkippableTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb.test_1."
+            ]
+          )
+        end
+
+        it "runs all tests in unskippable suite and sets forced run tag" do
+          expect(test_spans).to have(3).items
+          expect(test_spans).to have_tag_values_no_order(:status, ["pass", "pass", "skip"])
+
+          unskippable = test_spans.select { |span| span.get_tag("test.status") == "pass" }
+          expect(unskippable).to all have_test_tag(:itr_forced_run, "true")
+
+          expect(test_session_span).to have_test_tag(:itr_tests_skipped, "true")
+          expect(test_session_span).to have_test_tag(:itr_test_skipping_count, 1)
+        end
+      end
+
+      context "partially unskippable suite" do
+        before(:context) do
+          Minitest::Runnable.reset
+
+          class PartiallyUnskippableTest < Minitest::Test
+            datadog_itr_unskippable "test_1"
+
+            def test_1
+            end
+
+            def test_2
+            end
+          end
+        end
+
+        let(:itr_skippable_tests) do
+          Set.new(
+            [
+              "PartiallyUnskippableTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb.test_1.",
+              "PartiallyUnskippableTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb.test_2."
+            ]
+          )
+        end
+
+        it "runs unskippable test and sets forced run tag" do
+          expect(test_spans).to have(2).items
+          expect(test_spans).to have_tag_values_no_order(:status, ["pass", "skip"])
+
+          unskippable = test_spans.select { |span| span.get_tag("test.status") == "pass" }
+          expect(unskippable).to all have_test_tag(:itr_forced_run, "true")
+
+          expect(test_session_span).to have_test_tag(:itr_tests_skipped, "true")
+          expect(test_session_span).to have_test_tag(:itr_test_skipping_count, 1)
+        end
+      end
     end
   end
 end