instrument Minitest::Runnable to trace test suites for serial execution

lib/datadog/ci/contrib/minitest/hooks.rb

@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
+require_relative "suite"
 
 module Datadog
   module CI
@@ -15,10 +16,7 @@ def before_setup
 
             test_name = "#{class_name}##{name}"
 
-            source_location, = method(name).source_location
-            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
-
-            test_suite_name = "#{class_name} at #{source_file_path}"
+            test_suite_name = Suite.name(self.class, name)
             if parallel?
               test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
             end
@@ -43,6 +41,10 @@ def after_teardown
             when "."
               test_span.passed!
             when "E", "F"
+              test_suite_name = test_span.test_suite_name
+              test_suite = CI.active_test_suite(test_suite_name) if test_suite_name
+              test_suite.failed! if test_suite
+
               test_span.failed!(exception: failure)
             when "S"
               test_span.skipped!(reason: failure.message)

lib/datadog/ci/contrib/minitest/patcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "hooks"
+require_relative "runnable"
 
 module Datadog
   module CI
@@ -21,6 +22,7 @@ def patch
 
             ::Minitest::Test.include(Hooks)
             ::Minitest.include(Plugin)
+            ::Minitest::Runnable.include(Runnable)
 
             ::Minitest.extensions << "datadog_ci"
           end

lib/datadog/ci/contrib/minitest/runnable.rb

@@ -0,0 +1,45 @@
+require_relative "suite"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def run(*)
+              return super unless datadog_configuration[:enabled]
+              return super if parallel?
+              return super if runnable_methods.empty?
+
+              method = runnable_methods.first
+              test_suite_name = Suite.name(self, method)
+
+              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+              test_suite.passed! # will be overridden if any test fails
+
+              results = super
+
+              test_suite.finish
+
+              results
+            end
+
+            private
+
+            def parallel?
+              test_order == :parallel
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/minitest/suite.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        # Minitest integration constants
+        # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
+        module Suite
+          def self.name(klass, method_name)
+            source_location, = klass.instance_method(method_name).source_location
+            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
+
+            "#{klass.name} at #{source_file_path}"
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/minitest/runnable.rbs

@@ -0,0 +1,26 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods : ::Minitest::Runnable
+
+            def run: (*untyped) -> untyped
+
+            private
+
+            def parallel?: () -> bool
+
+            def datadog_configuration: () -> untyped
+
+            def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
+
+            def runnable_methods: () -> Array[String]
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/minitest/suite.rbs

@@ -0,0 +1,11 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Suite
+          def self.name: (untyped klass, String? method_name) -> ::String
+        end
+      end
+    end
+  end
+end

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -373,6 +373,10 @@ class SomeTest < Minitest::Test
           def test_pass
             assert true
           end
+
+          def test_pass_other
+            assert true
+          end
         end
       end
 
@@ -419,9 +423,35 @@ def test_pass
         )
       end
 
-      it "creates test span and connects it to the session and module" do
-        expect(test_spans.count).to eq(1)
+      it "creates a test suite span" do
+        expect(test_suite_span).not_to be_nil
 
+        expect(test_suite_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SUITE)
+        expect(test_suite_span.name).to eq("SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb")
+
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+          Datadog::CI::Ext::AppTypes::TYPE_TEST
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+          Datadog::CI::Ext::Test::TEST_TYPE
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+          Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+          Datadog::CI::Contrib::Minitest::Integration.version.to_s
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+      end
+
+      it "creates test spans and connects them to the session, module, and suite" do
+        expect(test_spans.count).to eq(2)
+
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
+          "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+        )
         expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
           Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
         )
@@ -432,8 +462,18 @@ def test_pass
           Datadog::CI::Ext::Test::Status::PASS
         )
 
-        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
-        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+        test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq
+        test_module_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID) }.uniq
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+
+        expect(test_session_ids.count).to eq(1)
+        expect(test_session_ids.first).to eq(test_session_span.id.to_s)
+
+        expect(test_module_ids.count).to eq(1)
+        expect(test_module_ids.first).to eq(test_module_span.id.to_s)
+
+        expect(test_suite_ids.count).to eq(1)
+        expect(test_suite_ids.first).to eq(test_suite_span.id.to_s)
       end
     end
 
@@ -460,6 +500,9 @@ def test_fail
         expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
           Datadog::CI::Ext::Test::Status::FAIL
         )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
       end
     end
 
@@ -501,10 +544,19 @@ class SomeSpec < Minitest::Spec
         )
       end
 
-      it "connects tests to different test suites" do
-        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.uniq
+      it "connects tests to different test suites (one per spec context)" do
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort
 
-        expect(test_suite_names.count).to eq(4)
+        expect(test_suite_ids).to have(4).items
+        expect(test_suite_names).to eq(
+          [
+            "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in context::deeper context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in other context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+          ]
+        )
       end
 
       it "connects tests to a single test session" do
@@ -601,6 +653,12 @@ def test_b_2
           expect(span.duration).to be < test_session_duration
         end
       end
+
+      it "creates test suite spans" do
+        skip("test suite spans for parallel execution pending")
+
+        expect(test_suite_spans).to have(4).items
+      end
     end
   end
 end

spec/datadog/ci/contrib/minitest/patcher_spec.rb

@@ -4,12 +4,29 @@
   describe ".patch" do
     subject!(:patch) { described_class.patch }
 
-    let(:test) { Minitest::Test }
-
-    context "is patched" do
+    context "Minitest::Test is patched" do
+      let(:test) { Minitest::Test }
       it "has a custom bases" do
         expect(test.ancestors).to include(Datadog::CI::Contrib::Minitest::Hooks)
       end
     end
+
+    context "Minitest::Runnable is patched" do
+      let(:runnable) { Minitest::Runnable }
+      it "has a custom bases" do
+        expect(runnable.ancestors).to include(Datadog::CI::Contrib::Minitest::Runnable)
+      end
+    end
+
+    context "Minitest includes plugin" do
+      let(:minitest) { Minitest }
+      it "has a custom bases" do
+        expect(minitest.ancestors).to include(Datadog::CI::Contrib::Minitest::Plugin)
+      end
+
+      it "has datadog_ci extension" do
+        expect(minitest.extensions).to include("datadog_ci")
+      end
+    end
   end
 end