From 5afea379066490593b622fb4591e1584f87dc78d Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 15 Dec 2023 15:03:05 +0100
Subject: [PATCH 01/11] add datadog ci plugin and reporter for minitest to
 track test session start/end events

---
 .standard_todo.yml                            |  2 +
 Steepfile                                     |  2 +
 .../minitest/configuration/settings.rb        |  2 +-
 lib/datadog/ci/contrib/minitest/ext.rb        | 10 ++-
 lib/datadog/ci/contrib/minitest/hooks.rb      |  4 +-
 lib/datadog/ci/contrib/minitest/patcher.rb    |  4 +
 lib/datadog/ci/contrib/minitest/plugin.rb     | 73 +++++++++++++++++
 sig/datadog/ci/contrib/minitest/ext.rbs       |  6 +-
 sig/datadog/ci/contrib/minitest/plugin.rbs    | 26 ++++++
 .../contrib/minitest/instrumentation_spec.rb  | 80 ++++++++++++++++++-
 vendor/rbs/weakref/0/weakref.rbs              |  6 ++
 11 files changed, 201 insertions(+), 14 deletions(-)
 create mode 100644 lib/datadog/ci/contrib/minitest/plugin.rb
 create mode 100644 sig/datadog/ci/contrib/minitest/plugin.rbs
 create mode 100644 vendor/rbs/weakref/0/weakref.rbs

diff --git a/.standard_todo.yml b/.standard_todo.yml
index d72dcc59..63304fc5 100644
--- a/.standard_todo.yml
+++ b/.standard_todo.yml
@@ -16,3 +16,5 @@ ignore:
       - Performance/UnfreezeString
   - Appraisals:
       - Style/Alias
+  - spec/datadog/ci/contrib/minitest/instrumentation_spec.rb:
+      - Lint/ConstantDefinitionInBlock
diff --git a/Steepfile b/Steepfile
index e6cdc0c9..dba07a3b 100644
--- a/Steepfile
+++ b/Steepfile
@@ -4,6 +4,7 @@ target :lib do
   check "lib"
 
   ignore "lib/datadog/ci/configuration/settings.rb"
+  ignore "lib/datadog/ci/contrib/minitest/plugin.rb"
 
   library "pathname"
   library "json"
@@ -20,4 +21,5 @@ target :lib do
   library "rspec"
   library "cucumber"
   library "msgpack"
+  library "weakref"
 end
diff --git a/lib/datadog/ci/contrib/minitest/configuration/settings.rb b/lib/datadog/ci/contrib/minitest/configuration/settings.rb
index 4a8e7c15..b3caa11c 100644
--- a/lib/datadog/ci/contrib/minitest/configuration/settings.rb
+++ b/lib/datadog/ci/contrib/minitest/configuration/settings.rb
@@ -19,7 +19,7 @@ class Settings < Datadog::CI::Contrib::Settings
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0
diff --git a/lib/datadog/ci/contrib/minitest/ext.rb b/lib/datadog/ci/contrib/minitest/ext.rb
index b45ab2a3..6fe194ce 100644
--- a/lib/datadog/ci/contrib/minitest/ext.rb
+++ b/lib/datadog/ci/contrib/minitest/ext.rb
@@ -7,13 +7,15 @@ module Minitest
         # Minitest integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "minitest"
           ENV_ENABLED = "DD_TRACE_MINITEST_ENABLED"
-          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
+
           FRAMEWORK = "minitest"
+
+          DEFAULT_SERVICE_NAME = "minitest"
+
+          # TODO: remove in 1.0
+          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
           OPERATION_NAME = "minitest.test"
-          SERVICE_NAME = "minitest"
-          TEST_TYPE = "test"
         end
       end
     end
diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 570354a8..104e09f0 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -24,7 +24,7 @@ def before_setup
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
               },
               service: configuration[:service_name]
             )
@@ -51,7 +51,7 @@ def after_teardown
           private
 
           def configuration
-            ::Datadog.configuration.ci[:minitest]
+            Datadog.configuration.ci[:minitest]
           end
         end
       end
diff --git a/lib/datadog/ci/contrib/minitest/patcher.rb b/lib/datadog/ci/contrib/minitest/patcher.rb
index b1b0d73b..e294d47a 100644
--- a/lib/datadog/ci/contrib/minitest/patcher.rb
+++ b/lib/datadog/ci/contrib/minitest/patcher.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "hooks"
+require_relative "plugin"
 
 module Datadog
   module CI
@@ -18,6 +19,9 @@ def target_version
 
           def patch
             ::Minitest::Test.include(Hooks)
+            ::Minitest.include(Plugin)
+
+            ::Minitest.extensions << "datadog_ci"
           end
         end
       end
diff --git a/lib/datadog/ci/contrib/minitest/plugin.rb b/lib/datadog/ci/contrib/minitest/plugin.rb
new file mode 100644
index 00000000..4e767d2a
--- /dev/null
+++ b/lib/datadog/ci/contrib/minitest/plugin.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "weakref"
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Plugin
+          def self.included(base)
+            base.extend(ClassMethods)
+          end
+
+          module ClassMethods
+            def plugin_datadog_ci_init(*)
+              return unless datadog_configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                },
+                service: datadog_configuration[:service_name]
+              )
+              CI.start_test_module(test_session.name)
+
+              # we create dynamic class here to avoid referencing ::Minitest constant
+              # in datadog-ci class definitions because Minitest is not always available
+              datadog_reporter_klass = Class.new(::Minitest::AbstractReporter) do
+                def initialize(reporter)
+                  # This creates circular reference as reporter holds reference to this reporter.
+                  # To make sure that reporter can be garbage collected, we use WeakRef.
+                  @reporter = WeakRef.new(reporter)
+                end
+
+                def report
+                  active_test_session = CI.active_test_session
+                  active_test_module = CI.active_test_module
+
+                  return unless @reporter.weakref_alive?
+                  return if active_test_session.nil? || active_test_module.nil?
+
+                  if @reporter.passed?
+                    active_test_module.passed!
+                    active_test_session.passed!
+                  else
+                    active_test_module.failed!
+                    active_test_session.failed!
+                  end
+
+                  active_test_module.finish
+                  active_test_session.finish
+                end
+              end
+
+              reporter.reporters << datadog_reporter_klass.new(reporter)
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/sig/datadog/ci/contrib/minitest/ext.rbs b/sig/datadog/ci/contrib/minitest/ext.rbs
index 9af544e4..8f4847c6 100644
--- a/sig/datadog/ci/contrib/minitest/ext.rbs
+++ b/sig/datadog/ci/contrib/minitest/ext.rbs
@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Minitest
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,9 +11,7 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          DEFAULT_SERVICE_NAME: String
         end
       end
     end
diff --git a/sig/datadog/ci/contrib/minitest/plugin.rbs b/sig/datadog/ci/contrib/minitest/plugin.rbs
new file mode 100644
index 00000000..c6dfd431
--- /dev/null
+++ b/sig/datadog/ci/contrib/minitest/plugin.rbs
@@ -0,0 +1,26 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Plugin
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods
+            attr_reader reporter: WeakRef
+            @reporter: WeakRef
+
+            def plugin_datadog_ci_init: (*untyped) -> (nil | untyped)
+
+            def initialize: (untyped reporter) -> void
+
+            def report: () -> (nil | untyped)
+
+            private
+
+            def datadog_configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index b875c90a..1ae8c9d1 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -2,7 +2,7 @@
 require "minitest"
 require "minitest/spec"
 
-RSpec.describe "Minitest hooks" do
+RSpec.describe "Minitest instrumentation" do
   include_context "CI mode activated" do
     let(:integration_name) { :minitest }
     let(:integration_options) { {service_name: "ltest"} }
@@ -34,7 +34,7 @@ def test_foo
       "spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
     )
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Contrib::Minitest::Ext::TEST_TYPE)
+    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK)
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
       Datadog::CI::Contrib::Minitest::Integration.version.to_s
@@ -352,4 +352,80 @@ def test_foo
       expect_skip
     end
   end
+
+  context "run minitest suite" do
+    before(:context) do
+      Minitest::Runnable.reset
+
+      class SomeTest < Minitest::Test
+        def test_pass
+          assert true
+        end
+      end
+    end
+
+    before do
+      Minitest.run([])
+    end
+
+    it "creates a test session span" do
+      expect(test_session_span).not_to be_nil
+      expect(test_session_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION)
+      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+        Datadog::CI::Ext::AppTypes::TYPE_TEST
+      )
+      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+        Datadog::CI::Ext::Test::TEST_TYPE
+      )
+      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+      )
+      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+        Datadog::CI::Contrib::Minitest::Integration.version.to_s
+      )
+      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::PASS
+      )
+    end
+
+    it "creates a test module span" do
+      expect(test_module_span).not_to be_nil
+
+      expect(test_module_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_MODULE)
+      expect(test_module_span.name).to eq(test_command)
+
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+        Datadog::CI::Ext::AppTypes::TYPE_TEST
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+        Datadog::CI::Ext::Test::TEST_TYPE
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+        Datadog::CI::Contrib::Minitest::Integration.version.to_s
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::PASS
+      )
+    end
+
+    it "creates test span and connects it to the session and module" do
+      expect(test_spans.count).to eq(1)
+
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+      )
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+        Datadog::CI::Contrib::Minitest::Integration.version.to_s
+      )
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::PASS
+      )
+
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+    end
+  end
 end
diff --git a/vendor/rbs/weakref/0/weakref.rbs b/vendor/rbs/weakref/0/weakref.rbs
new file mode 100644
index 00000000..f53b31a2
--- /dev/null
+++ b/vendor/rbs/weakref/0/weakref.rbs
@@ -0,0 +1,6 @@
+# as rbs misses delegator support, we make WeakRef a subclass of CompositeReporter
+# for our current use case
+class WeakRef < Minitest::CompositeReporter
+  def initialize: (untyped obj) -> untyped
+  def weakref_alive?: () -> bool
+end

From 59c0ebd3cbd2502f0e3c4227ed860fb17ae0c841 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Wed, 27 Dec 2023 16:56:52 +0100
Subject: [PATCH 02/11] require minitest plugin only when patching and avoid
 issues with referencing Minitest

---
 Steepfile                                     |  1 -
 lib/datadog/ci/contrib/minitest/patcher.rb    |  3 +-
 lib/datadog/ci/contrib/minitest/plugin.rb     | 60 +++++++++----------
 lib/datadog/ci/contrib/rspec/integration.rb   |  2 +-
 lib/datadog/ci/contrib/settings.rb            |  2 +-
 sig/datadog/ci/contrib/minitest/plugin.rbs    |  9 ++-
 .../contrib/minitest/instrumentation_spec.rb  |  2 +-
 7 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/Steepfile b/Steepfile
index dba07a3b..8707ad60 100644
--- a/Steepfile
+++ b/Steepfile
@@ -4,7 +4,6 @@ target :lib do
   check "lib"
 
   ignore "lib/datadog/ci/configuration/settings.rb"
-  ignore "lib/datadog/ci/contrib/minitest/plugin.rb"
 
   library "pathname"
   library "json"
diff --git a/lib/datadog/ci/contrib/minitest/patcher.rb b/lib/datadog/ci/contrib/minitest/patcher.rb
index e294d47a..0b51bec1 100644
--- a/lib/datadog/ci/contrib/minitest/patcher.rb
+++ b/lib/datadog/ci/contrib/minitest/patcher.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "hooks"
-require_relative "plugin"
 
 module Datadog
   module CI
@@ -18,6 +17,8 @@ def target_version
           end
 
           def patch
+            require_relative "plugin"
+
             ::Minitest::Test.include(Hooks)
             ::Minitest.include(Plugin)
 
diff --git a/lib/datadog/ci/contrib/minitest/plugin.rb b/lib/datadog/ci/contrib/minitest/plugin.rb
index 4e767d2a..0df4892c 100644
--- a/lib/datadog/ci/contrib/minitest/plugin.rb
+++ b/lib/datadog/ci/contrib/minitest/plugin.rb
@@ -14,6 +14,35 @@ def self.included(base)
             base.extend(ClassMethods)
           end
 
+          class DatadogReporter < ::Minitest::AbstractReporter
+            def initialize(minitest_reporter)
+              # This creates circular reference as minitest_reporter also holds reference to DatadogReporter.
+              # To make sure that minitest_reporter can be garbage collected, we use WeakRef.
+              @reporter = WeakRef.new(minitest_reporter)
+            end
+
+            def report
+              active_test_session = CI.active_test_session
+              active_test_module = CI.active_test_module
+
+              return unless @reporter.weakref_alive?
+              return if active_test_session.nil? || active_test_module.nil?
+
+              if @reporter.passed?
+                active_test_module.passed!
+                active_test_session.passed!
+              else
+                active_test_module.failed!
+                active_test_session.failed!
+              end
+
+              active_test_module.finish
+              active_test_session.finish
+
+              nil
+            end
+          end
+
           module ClassMethods
             def plugin_datadog_ci_init(*)
               return unless datadog_configuration[:enabled]
@@ -28,36 +57,7 @@ def plugin_datadog_ci_init(*)
               )
               CI.start_test_module(test_session.name)
 
-              # we create dynamic class here to avoid referencing ::Minitest constant
-              # in datadog-ci class definitions because Minitest is not always available
-              datadog_reporter_klass = Class.new(::Minitest::AbstractReporter) do
-                def initialize(reporter)
-                  # This creates circular reference as reporter holds reference to this reporter.
-                  # To make sure that reporter can be garbage collected, we use WeakRef.
-                  @reporter = WeakRef.new(reporter)
-                end
-
-                def report
-                  active_test_session = CI.active_test_session
-                  active_test_module = CI.active_test_module
-
-                  return unless @reporter.weakref_alive?
-                  return if active_test_session.nil? || active_test_module.nil?
-
-                  if @reporter.passed?
-                    active_test_module.passed!
-                    active_test_session.passed!
-                  else
-                    active_test_module.failed!
-                    active_test_session.failed!
-                  end
-
-                  active_test_module.finish
-                  active_test_session.finish
-                end
-              end
-
-              reporter.reporters << datadog_reporter_klass.new(reporter)
+              reporter.reporters << DatadogReporter.new(reporter)
             end
 
             private
diff --git a/lib/datadog/ci/contrib/rspec/integration.rb b/lib/datadog/ci/contrib/rspec/integration.rb
index dbd625cc..339cf2be 100644
--- a/lib/datadog/ci/contrib/rspec/integration.rb
+++ b/lib/datadog/ci/contrib/rspec/integration.rb
@@ -22,7 +22,7 @@ def self.version
           end
 
           def self.loaded?
-            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && \
+            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? &&
               !defined?(::RSpec::Core::Example).nil?
           end
 
diff --git a/lib/datadog/ci/contrib/settings.rb b/lib/datadog/ci/contrib/settings.rb
index 2947c8c0..77097c22 100644
--- a/lib/datadog/ci/contrib/settings.rb
+++ b/lib/datadog/ci/contrib/settings.rb
@@ -27,7 +27,7 @@ def [](name)
         end
 
         def []=(name, value)
-          respond_to?("#{name}=") ? send("#{name}=", value) : set_option(name, value)
+          respond_to?(:"#{name}=") ? send(:"#{name}=", value) : set_option(name, value)
         end
       end
     end
diff --git a/sig/datadog/ci/contrib/minitest/plugin.rbs b/sig/datadog/ci/contrib/minitest/plugin.rbs
index c6dfd431..b2c73996 100644
--- a/sig/datadog/ci/contrib/minitest/plugin.rbs
+++ b/sig/datadog/ci/contrib/minitest/plugin.rbs
@@ -5,16 +5,21 @@ module Datadog
         module Plugin
           def self.included: (untyped base) -> untyped
 
-          module ClassMethods
-            attr_reader reporter: WeakRef
+          class DatadogReporter < ::Minitest::AbstractReporter
             @reporter: WeakRef
 
+            def initialize: (Minitest::AbstractReporter reporter) -> void
+          end
+
+          module ClassMethods
             def plugin_datadog_ci_init: (*untyped) -> (nil | untyped)
 
             def initialize: (untyped reporter) -> void
 
             def report: () -> (nil | untyped)
 
+            def reporter: () -> Minitest::CompositeReporter
+
             private
 
             def datadog_configuration: () -> untyped
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index 1ae8c9d1..08591a64 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -53,7 +53,7 @@ def self.name
       end
 
       num_tests.times do |i|
-        define_method("test_#{i}") {}
+        define_method(:"test_#{i}") {}
       end
     end
 

From f1c50f447a61ed04778fad45ca7f99b50501ab46 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Thu, 28 Dec 2023 15:16:44 +0100
Subject: [PATCH 03/11] Minitest::Spec example for instrumentation tests

---
 .../contrib/minitest/instrumentation_spec.rb  | 215 +++++++++++++-----
 1 file changed, 152 insertions(+), 63 deletions(-)

diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index 08591a64..046c45e4 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -1,7 +1,14 @@
 require "time"
+
 require "minitest"
 require "minitest/spec"
 
+# minitest adds `describe` method to Kernel, which conflicts with RSpec.
+# here we define `minitest_describe` method to avoid this conflict.
+module Kernel
+  alias_method :minitest_describe, :describe
+end
+
 RSpec.describe "Minitest instrumentation" do
   include_context "CI mode activated" do
     let(:integration_name) { :minitest }
@@ -354,78 +361,160 @@ def test_foo
   end
 
   context "run minitest suite" do
-    before(:context) do
-      Minitest::Runnable.reset
+    before do
+      Minitest.run([])
+    end
+
+    context "single test passed" do
+      before(:context) do
+        Minitest::Runnable.reset
 
-      class SomeTest < Minitest::Test
-        def test_pass
-          assert true
+        class SomeTest < Minitest::Test
+          def test_pass
+            assert true
+          end
         end
       end
-    end
 
-    before do
-      Minitest.run([])
-    end
+      it "creates a test session span" do
+        expect(test_session_span).not_to be_nil
+        expect(test_session_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION)
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+          Datadog::CI::Ext::AppTypes::TYPE_TEST
+        )
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+          Datadog::CI::Ext::Test::TEST_TYPE
+        )
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+          Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+        )
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+          Datadog::CI::Contrib::Minitest::Integration.version.to_s
+        )
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+      end
+
+      it "creates a test module span" do
+        expect(test_module_span).not_to be_nil
+
+        expect(test_module_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_MODULE)
+        expect(test_module_span.name).to eq(test_command)
+
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+          Datadog::CI::Ext::AppTypes::TYPE_TEST
+        )
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+          Datadog::CI::Ext::Test::TEST_TYPE
+        )
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+          Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+        )
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+          Datadog::CI::Contrib::Minitest::Integration.version.to_s
+        )
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+      end
 
-    it "creates a test session span" do
-      expect(test_session_span).not_to be_nil
-      expect(test_session_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION)
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
-      )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
-      )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
-        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
-      )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
-        Datadog::CI::Contrib::Minitest::Integration.version.to_s
-      )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
-        Datadog::CI::Ext::Test::Status::PASS
-      )
+      it "creates test span and connects it to the session and module" do
+        expect(test_spans.count).to eq(1)
+
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+          Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+        )
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+          Datadog::CI::Contrib::Minitest::Integration.version.to_s
+        )
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+      end
     end
 
-    it "creates a test module span" do
-      expect(test_module_span).not_to be_nil
-
-      expect(test_module_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_MODULE)
-      expect(test_module_span.name).to eq(test_command)
-
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
-        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
-        Datadog::CI::Contrib::Minitest::Integration.version.to_s
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
-        Datadog::CI::Ext::Test::Status::PASS
-      )
+    context "single test failed" do
+      before(:context) do
+        Minitest::Runnable.reset
+
+        class SomeFailedTest < Minitest::Test
+          def test_fail
+            assert false
+          end
+        end
+      end
+
+      it "traces test, test session, test module with failed status" do
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeFailedTest#test_fail")
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
+
+        expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
+        expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
+      end
     end
 
-    it "creates test span and connects it to the session and module" do
-      expect(test_spans.count).to eq(1)
-
-      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
-        Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
-      )
-      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
-        Datadog::CI::Contrib::Minitest::Integration.version.to_s
-      )
-      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
-        Datadog::CI::Ext::Test::Status::PASS
-      )
-
-      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
-      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+    context "using Minitest::Spec" do
+      before(:context) do
+        Minitest::Runnable.reset
+
+        class SomeSpec < Minitest::Spec
+          it "does not fail" do
+          end
+
+          minitest_describe "in context" do
+            it "does not fail" do
+            end
+
+            minitest_describe "deeper context" do
+              it "does not fail" do
+              end
+            end
+          end
+
+          minitest_describe "in other context" do
+            it "does not fail" do
+            end
+          end
+        end
+      end
+
+      it "traces tests with unique names" do
+        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
+
+        expect(test_names).to eq(
+          [
+            "SomeSpec#test_0001_does not fail",
+            "in context#test_0001_does not fail",
+            "in context::deeper context#test_0001_does not fail",
+            "in other context#test_0001_does not fail"
+          ]
+        )
+      end
+
+      it "connects tests to different test suites" do
+        skip("pending fix for minitest/spec")
+
+        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.uniq
+
+        expect(test_suite_names.count).to eq(4)
+      end
+
+      it "connects tests to a single test session" do
+        test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq
+
+        expect(test_session_ids.count).to eq(1)
+        expect(test_session_ids.first).to eq(test_session_span.id.to_s)
+      end
     end
   end
 end

From 1b5d8e99cef9c376be722bcf109fa9113df147f7 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Thu, 28 Dec 2023 16:16:37 +0100
Subject: [PATCH 04/11] minitest example with parallel executor

---
 .../contrib/minitest/instrumentation_spec.rb  | 78 +++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index 046c45e4..864eb6f1 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -516,5 +516,83 @@ class SomeSpec < Minitest::Spec
         expect(test_session_ids.first).to eq(test_session_span.id.to_s)
       end
     end
+
+    context "using parallel executor" do
+      before(:context) do
+        require "minitest/hell"
+
+        Minitest::Runnable.reset
+
+        class ParallelTest < Minitest::Test
+          parallelize_me!
+        end
+
+        class TestA < ParallelTest
+          def test_a_1
+            Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id)
+            sleep 0.1
+          end
+
+          def test_a_2
+            Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id)
+            sleep 0.1
+          end
+        end
+
+        class TestB < ParallelTest
+          def test_b_1
+            Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id)
+            sleep 0.1
+          end
+
+          def test_b_2
+            Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id)
+            sleep 0.1
+          end
+        end
+      end
+
+      it "traces all tests correctly" do
+        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
+        test_threads = test_spans.map { |span| span.get_tag("minitest_thread") }.uniq
+
+        expect(test_names).to eq(
+          [
+            "TestA#test_a_1",
+            "TestA#test_a_2",
+            "TestB#test_b_1",
+            "TestB#test_b_2"
+          ]
+        )
+
+        # make sure that tests were executed concurrently
+        # note that this test could be flaky
+        expect(test_threads.count).to be > 1
+      end
+
+      it "connects tests to a single test session and a single test module" do
+        test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq
+        test_module_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID) }.uniq
+
+        expect(test_session_ids.count).to eq(1)
+        expect(test_session_ids.first).to eq(test_session_span.id.to_s)
+
+        expect(test_module_ids.count).to eq(1)
+        expect(test_module_ids.first).to eq(test_module_span.id.to_s)
+      end
+
+      it "correctly tracks test and session durations" do
+        test_session_duration = test_session_span.duration
+
+        test_durations_sum = test_spans.map { |span| span.duration }.sum
+        # with parallel execution test durations sum should be greater than test session duration
+        expect(test_durations_sum).to be > test_session_duration
+
+        # but each individual test duration should be less than test session duration
+        test_spans.each do |span|
+          expect(span.duration).to be < test_session_duration
+        end
+      end
+    end
   end
 end

From 5c53f856fedbcd534798c881ffeb5d3807b24870 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 29 Dec 2023 10:46:59 +0100
Subject: [PATCH 05/11] change test suite name definition for Minitest to
 reflect Minitest::Spec and parallel execution of test cases

---
 lib/datadog/ci/contrib/minitest/hooks.rb      | 21 +++++++++++++------
 sig/datadog/ci/contrib/minitest/hooks.rbs     |  6 +++++-
 .../contrib/minitest/instrumentation_spec.rb  | 16 ++++++++++----
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 104e09f0..767b0b30 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -11,22 +11,27 @@ module Minitest
         module Hooks
           def before_setup
             super
-            return unless configuration[:enabled]
+            return unless datadog_configuration[:enabled]
 
             test_name = "#{class_name}##{name}"
 
-            path, = method(name).source_location
-            test_suite = Pathname.new(path.to_s).relative_path_from(Pathname.pwd).to_s
+            source_location, = method(name).source_location
+            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
+
+            test_suite_name = "#{class_name} at #{source_file_path}"
+            if parallel?
+              test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
+            end
 
             CI.start_test(
               test_name,
-              test_suite,
+              test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
                 CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
               },
-              service: configuration[:service_name]
+              service: datadog_configuration[:service_name]
             )
           end
 
@@ -50,7 +55,11 @@ def after_teardown
 
           private
 
-          def configuration
+          def parallel?
+            self.class.test_order == :parallel
+          end
+
+          def datadog_configuration
             Datadog.configuration.ci[:minitest]
           end
         end
diff --git a/sig/datadog/ci/contrib/minitest/hooks.rbs b/sig/datadog/ci/contrib/minitest/hooks.rbs
index 94e1fe62..7e4b1037 100644
--- a/sig/datadog/ci/contrib/minitest/hooks.rbs
+++ b/sig/datadog/ci/contrib/minitest/hooks.rbs
@@ -11,7 +11,11 @@ module Datadog
 
           private
 
-          def configuration: () -> untyped
+          def parallel?: () -> bool
+
+          def datadog_configuration: () -> untyped
+
+          def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha)
         end
       end
     end
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index 864eb6f1..e753d4bf 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -38,7 +38,7 @@ def test_foo
     expect(span.service).to eq("ltest")
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeTest#test_foo")
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
-      "spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+      "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
     )
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
@@ -89,7 +89,7 @@ def self.name
     expect(span.service).to eq("ltest")
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeSpec##{method_name}")
     expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
-      "spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+      "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
     )
   end
 
@@ -502,8 +502,6 @@ class SomeSpec < Minitest::Spec
       end
 
       it "connects tests to different test suites" do
-        skip("pending fix for minitest/spec")
-
         test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.uniq
 
         expect(test_suite_names.count).to eq(4)
@@ -554,6 +552,7 @@ def test_b_2
 
       it "traces all tests correctly" do
         test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
+        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort
         test_threads = test_spans.map { |span| span.get_tag("minitest_thread") }.uniq
 
         expect(test_names).to eq(
@@ -565,6 +564,15 @@ def test_b_2
           ]
         )
 
+        expect(test_suite_names).to eq(
+          [
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
+          ]
+        )
+
         # make sure that tests were executed concurrently
         # note that this test could be flaky
         expect(test_threads.count).to be > 1

From 17e5dc616c6ca8e76e70b1e4d9e0314388c5c828 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 29 Dec 2023 14:32:40 +0100
Subject: [PATCH 06/11] instrument Minitest::Runnable to trace test suites for
 serial execution

---
 lib/datadog/ci/contrib/minitest/hooks.rb      | 10 +--
 lib/datadog/ci/contrib/minitest/patcher.rb    |  2 +
 lib/datadog/ci/contrib/minitest/runnable.rb   | 45 ++++++++++++
 lib/datadog/ci/contrib/minitest/suite.rb      | 20 ++++++
 sig/datadog/ci/contrib/minitest/runnable.rbs  | 26 +++++++
 sig/datadog/ci/contrib/minitest/suite.rbs     | 11 +++
 .../contrib/minitest/instrumentation_spec.rb  | 72 +++++++++++++++++--
 .../ci/contrib/minitest/patcher_spec.rb       | 23 +++++-
 8 files changed, 195 insertions(+), 14 deletions(-)
 create mode 100644 lib/datadog/ci/contrib/minitest/runnable.rb
 create mode 100644 lib/datadog/ci/contrib/minitest/suite.rb
 create mode 100644 sig/datadog/ci/contrib/minitest/runnable.rbs
 create mode 100644 sig/datadog/ci/contrib/minitest/suite.rbs

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 767b0b30..33b48a4a 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
+require_relative "suite"
 
 module Datadog
   module CI
@@ -15,10 +16,7 @@ def before_setup
 
             test_name = "#{class_name}##{name}"
 
-            source_location, = method(name).source_location
-            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
-
-            test_suite_name = "#{class_name} at #{source_file_path}"
+            test_suite_name = Suite.name(self.class, name)
             if parallel?
               test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
             end
@@ -43,6 +41,10 @@ def after_teardown
             when "."
               test_span.passed!
             when "E", "F"
+              test_suite_name = test_span.test_suite_name
+              test_suite = CI.active_test_suite(test_suite_name) if test_suite_name
+              test_suite.failed! if test_suite
+
               test_span.failed!(exception: failure)
             when "S"
               test_span.skipped!(reason: failure.message)
diff --git a/lib/datadog/ci/contrib/minitest/patcher.rb b/lib/datadog/ci/contrib/minitest/patcher.rb
index 0b51bec1..6393870b 100644
--- a/lib/datadog/ci/contrib/minitest/patcher.rb
+++ b/lib/datadog/ci/contrib/minitest/patcher.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "hooks"
+require_relative "runnable"
 
 module Datadog
   module CI
@@ -21,6 +22,7 @@ def patch
 
             ::Minitest::Test.include(Hooks)
             ::Minitest.include(Plugin)
+            ::Minitest::Runnable.include(Runnable)
 
             ::Minitest.extensions << "datadog_ci"
           end
diff --git a/lib/datadog/ci/contrib/minitest/runnable.rb b/lib/datadog/ci/contrib/minitest/runnable.rb
new file mode 100644
index 00000000..1b416fcd
--- /dev/null
+++ b/lib/datadog/ci/contrib/minitest/runnable.rb
@@ -0,0 +1,45 @@
+require_relative "suite"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def run(*)
+              return super unless datadog_configuration[:enabled]
+              return super if parallel?
+              return super if runnable_methods.empty?
+
+              method = runnable_methods.first
+              test_suite_name = Suite.name(self, method)
+
+              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+              test_suite.passed! # will be overridden if any test fails
+
+              results = super
+
+              test_suite.finish
+
+              results
+            end
+
+            private
+
+            def parallel?
+              test_order == :parallel
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/datadog/ci/contrib/minitest/suite.rb b/lib/datadog/ci/contrib/minitest/suite.rb
new file mode 100644
index 00000000..c675384f
--- /dev/null
+++ b/lib/datadog/ci/contrib/minitest/suite.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        # Minitest integration constants
+        # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
+        module Suite
+          def self.name(klass, method_name)
+            source_location, = klass.instance_method(method_name).source_location
+            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
+
+            "#{klass.name} at #{source_file_path}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/sig/datadog/ci/contrib/minitest/runnable.rbs b/sig/datadog/ci/contrib/minitest/runnable.rbs
new file mode 100644
index 00000000..09b724f6
--- /dev/null
+++ b/sig/datadog/ci/contrib/minitest/runnable.rbs
@@ -0,0 +1,26 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods : ::Minitest::Runnable
+
+            def run: (*untyped) -> untyped
+
+            private
+
+            def parallel?: () -> bool
+
+            def datadog_configuration: () -> untyped
+
+            def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
+
+            def runnable_methods: () -> Array[String]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/sig/datadog/ci/contrib/minitest/suite.rbs b/sig/datadog/ci/contrib/minitest/suite.rbs
new file mode 100644
index 00000000..e87a538a
--- /dev/null
+++ b/sig/datadog/ci/contrib/minitest/suite.rbs
@@ -0,0 +1,11 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Suite
+          def self.name: (untyped klass, String? method_name) -> ::String
+        end
+      end
+    end
+  end
+end
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index e753d4bf..4cf58d6b 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -373,6 +373,10 @@ class SomeTest < Minitest::Test
           def test_pass
             assert true
           end
+
+          def test_pass_other
+            assert true
+          end
         end
       end
 
@@ -419,9 +423,35 @@ def test_pass
         )
       end
 
-      it "creates test span and connects it to the session and module" do
-        expect(test_spans.count).to eq(1)
+      it "creates a test suite span" do
+        expect(test_suite_span).not_to be_nil
 
+        expect(test_suite_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SUITE)
+        expect(test_suite_span.name).to eq("SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb")
+
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+          Datadog::CI::Ext::AppTypes::TYPE_TEST
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+          Datadog::CI::Ext::Test::TEST_TYPE
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+          Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+          Datadog::CI::Contrib::Minitest::Integration.version.to_s
+        )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+      end
+
+      it "creates test spans and connects them to the session, module, and suite" do
+        expect(test_spans.count).to eq(2)
+
+        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
+          "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+        )
         expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
           Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
         )
@@ -432,8 +462,18 @@ def test_pass
           Datadog::CI::Ext::Test::Status::PASS
         )
 
-        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
-        expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+        test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq
+        test_module_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID) }.uniq
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+
+        expect(test_session_ids.count).to eq(1)
+        expect(test_session_ids.first).to eq(test_session_span.id.to_s)
+
+        expect(test_module_ids.count).to eq(1)
+        expect(test_module_ids.first).to eq(test_module_span.id.to_s)
+
+        expect(test_suite_ids.count).to eq(1)
+        expect(test_suite_ids.first).to eq(test_suite_span.id.to_s)
       end
     end
 
@@ -460,6 +500,9 @@ def test_fail
         expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
           Datadog::CI::Ext::Test::Status::FAIL
         )
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
       end
     end
 
@@ -501,10 +544,19 @@ class SomeSpec < Minitest::Spec
         )
       end
 
-      it "connects tests to different test suites" do
-        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.uniq
+      it "connects tests to different test suites (one per spec context)" do
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort
 
-        expect(test_suite_names.count).to eq(4)
+        expect(test_suite_ids).to have(4).items
+        expect(test_suite_names).to eq(
+          [
+            "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in context::deeper context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb",
+            "in other context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
+          ]
+        )
       end
 
       it "connects tests to a single test session" do
@@ -601,6 +653,12 @@ def test_b_2
           expect(span.duration).to be < test_session_duration
         end
       end
+
+      it "creates test suite spans" do
+        skip("test suite spans for parallel execution pending")
+
+        expect(test_suite_spans).to have(4).items
+      end
     end
   end
 end
diff --git a/spec/datadog/ci/contrib/minitest/patcher_spec.rb b/spec/datadog/ci/contrib/minitest/patcher_spec.rb
index 313c144f..1f47fc7a 100644
--- a/spec/datadog/ci/contrib/minitest/patcher_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/patcher_spec.rb
@@ -4,12 +4,29 @@
   describe ".patch" do
     subject!(:patch) { described_class.patch }
 
-    let(:test) { Minitest::Test }
-
-    context "is patched" do
+    context "Minitest::Test is patched" do
+      let(:test) { Minitest::Test }
       it "has a custom bases" do
         expect(test.ancestors).to include(Datadog::CI::Contrib::Minitest::Hooks)
       end
     end
+
+    context "Minitest::Runnable is patched" do
+      let(:runnable) { Minitest::Runnable }
+      it "has a custom bases" do
+        expect(runnable.ancestors).to include(Datadog::CI::Contrib::Minitest::Runnable)
+      end
+    end
+
+    context "Minitest includes plugin" do
+      let(:minitest) { Minitest }
+      it "has a custom bases" do
+        expect(minitest.ancestors).to include(Datadog::CI::Contrib::Minitest::Plugin)
+      end
+
+      it "has datadog_ci extension" do
+        expect(minitest.extensions).to include("datadog_ci")
+      end
+    end
   end
 end

From addd8c7ab5308be23614629b2e285dd56b9a5c06 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 29 Dec 2023 16:27:21 +0100
Subject: [PATCH 07/11] track test suite per test when executing in parallel
 mode

---
 lib/datadog/ci/contrib/minitest/hooks.rb      | 47 +++++++---
 lib/datadog/ci/contrib/minitest/runnable.rb   |  3 +-
 lib/datadog/ci/span.rb                        | 24 +++++
 lib/datadog/ci/test.rb                        |  8 ++
 sig/datadog/ci/contrib/minitest/hooks.rbs     |  6 ++
 sig/datadog/ci/span.rbs                       |  8 ++
 sig/datadog/ci/test.rbs                       |  1 +
 .../contrib/minitest/instrumentation_spec.rb  | 39 ++++----
 spec/datadog/ci/span_spec.rb                  | 88 +++++++++++++++++++
 spec/datadog/ci/test_spec.rb                  | 20 +++++
 10 files changed, 210 insertions(+), 34 deletions(-)

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 33b48a4a..0d560e92 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -19,6 +19,9 @@ def before_setup
             test_suite_name = Suite.name(self.class, name)
             if parallel?
               test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
+
+              # for parallel execution we need to start a new test suite for each test
+              CI.start_test_suite(test_suite_name)
             end
 
             CI.start_test(
@@ -37,26 +40,44 @@ def after_teardown
             test_span = CI.active_test
             return super unless test_span
 
-            case result_code
-            when "."
-              test_span.passed!
-            when "E", "F"
-              test_suite_name = test_span.test_suite_name
-              test_suite = CI.active_test_suite(test_suite_name) if test_suite_name
-              test_suite.failed! if test_suite
-
-              test_span.failed!(exception: failure)
-            when "S"
-              test_span.skipped!(reason: failure.message)
+            finish_test(test_span, result_code)
+            if parallel?
+              finish_test_suite(test_span.test_suite, result_code)
             end
 
-            test_span.finish
-
             super
           end
 
           private
 
+          def finish_test(test_span, result_code)
+            finish_with_result(test_span, result_code)
+
+            # mark test suite as failed if any test failed
+            if test_span.failed?
+              test_suite = test_span.test_suite
+              test_suite.failed! if test_suite
+            end
+          end
+
+          def finish_test_suite(test_suite, result_code)
+            return unless test_suite
+
+            finish_with_result(test_suite, result_code)
+          end
+
+          def finish_with_result(span, result_code)
+            case result_code
+            when "."
+              span.passed!
+            when "E", "F"
+              span.failed!(exception: failure)
+            when "S"
+              span.skipped!(reason: failure.message)
+            end
+            span.finish
+          end
+
           def parallel?
             self.class.test_order == :parallel
           end
diff --git a/lib/datadog/ci/contrib/minitest/runnable.rb b/lib/datadog/ci/contrib/minitest/runnable.rb
index 1b416fcd..f1903de0 100644
--- a/lib/datadog/ci/contrib/minitest/runnable.rb
+++ b/lib/datadog/ci/contrib/minitest/runnable.rb
@@ -13,9 +13,10 @@ module ClassMethods
             def run(*)
               return super unless datadog_configuration[:enabled]
               return super if parallel?
-              return super if runnable_methods.empty?
 
               method = runnable_methods.first
+              return super if method.nil?
+
               test_suite_name = Suite.name(self, method)
 
               test_suite = Datadog::CI.start_test_suite(test_suite_name)
diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb
index 08f8cffe..b038611e 100644
--- a/lib/datadog/ci/span.rb
+++ b/lib/datadog/ci/span.rb
@@ -36,6 +36,30 @@ def span_type
         tracer_span.type
       end
 
+      # Checks whether span status is not set yet.
+      # @return [bool] true if span does not have status
+      def undefined?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS).nil?
+      end
+
+      # Checks whether span status is PASS.
+      # @return [bool] true if span status is PASS
+      def passed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::PASS
+      end
+
+      # Checks whether span status is FAIL.
+      # @return [bool] true if span status is FAIL
+      def failed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::FAIL
+      end
+
+      # Checks whether span status is SKIP.
+      # @return [bool] true if span status is SKIP
+      def skipped?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::SKIP
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!
diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb
index 4550b5be..d537c7fa 100644
--- a/lib/datadog/ci/test.rb
+++ b/lib/datadog/ci/test.rb
@@ -21,6 +21,14 @@ def finish
         CI.deactivate_test(self)
       end
 
+      # Running test suite that this test is part of (if any).
+      # @return [Datadog::CI::TestSuite] the test suite this test belongs to
+      # @return [nil] if the test suite is not found
+      def test_suite
+        suite_name = test_suite_name
+        CI.active_test_suite(suite_name) if suite_name
+      end
+
       # Span id of the running test suite this test belongs to.
       # @return [String] the span id of the test suite.
       def test_suite_id
diff --git a/sig/datadog/ci/contrib/minitest/hooks.rbs b/sig/datadog/ci/contrib/minitest/hooks.rbs
index 7e4b1037..ccc07f7f 100644
--- a/sig/datadog/ci/contrib/minitest/hooks.rbs
+++ b/sig/datadog/ci/contrib/minitest/hooks.rbs
@@ -15,6 +15,12 @@ module Datadog
 
           def datadog_configuration: () -> untyped
 
+          def finish_test: (Datadog::CI::Test test_span, String result_code) -> void
+
+          def finish_test_suite: (Datadog::CI::TestSuite? test_suite, String result_code) -> void
+
+          def finish_with_result: (Datadog::CI::Span span, String result_code) -> void
+
           def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha)
         end
       end
diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs
index 6265f0d6..b3a7a5a0 100644
--- a/sig/datadog/ci/span.rbs
+++ b/sig/datadog/ci/span.rbs
@@ -13,6 +13,14 @@ module Datadog
 
       def service: () -> String
 
+      def undefined?: () -> bool
+
+      def passed?: () -> bool
+
+      def failed?: () -> bool
+
+      def skipped?: () -> bool
+
       def passed!: () -> void
 
       def failed!: (?exception: untyped?) -> void
diff --git a/sig/datadog/ci/test.rbs b/sig/datadog/ci/test.rbs
index d14485ec..0a685590 100644
--- a/sig/datadog/ci/test.rbs
+++ b/sig/datadog/ci/test.rbs
@@ -2,6 +2,7 @@ module Datadog
   module CI
     class Test < Span
       def finish: () -> void
+      def test_suite: () -> Datadog::CI::TestSuite?
       def test_suite_id: () -> String?
       def test_suite_name: () -> String?
       def test_module_id: () -> String?
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index 4cf58d6b..bca20dd3 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -569,8 +569,6 @@ class SomeSpec < Minitest::Spec
 
     context "using parallel executor" do
       before(:context) do
-        require "minitest/hell"
-
         Minitest::Runnable.reset
 
         class ParallelTest < Minitest::Test
@@ -602,11 +600,14 @@ def test_b_2
         end
       end
 
-      it "traces all tests correctly" do
-        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
-        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort
+      it "traces all tests correctly, assigning a separate test suite to each of them" do
         test_threads = test_spans.map { |span| span.get_tag("minitest_thread") }.uniq
 
+        # make sure that tests were executed concurrently
+        # note that this test could be flaky
+        expect(test_threads.count).to be > 1
+
+        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
         expect(test_names).to eq(
           [
             "TestA#test_a_1",
@@ -616,18 +617,8 @@ def test_b_2
           ]
         )
 
-        expect(test_suite_names).to eq(
-          [
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
-          ]
-        )
-
-        # make sure that tests were executed concurrently
-        # note that this test could be flaky
-        expect(test_threads.count).to be > 1
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+        expect(test_suite_ids).to have(4).items
       end
 
       it "connects tests to a single test session and a single test module" do
@@ -648,16 +639,24 @@ def test_b_2
         # with parallel execution test durations sum should be greater than test session duration
         expect(test_durations_sum).to be > test_session_duration
 
-        # but each individual test duration should be less than test session duration
+        # each individual test duration should be less than test session duration
         test_spans.each do |span|
           expect(span.duration).to be < test_session_duration
         end
       end
 
       it "creates test suite spans" do
-        skip("test suite spans for parallel execution pending")
-
         expect(test_suite_spans).to have(4).items
+
+        test_suite_names = test_suite_spans.map { |span| span.name }.sort
+        expect(test_suite_names).to eq(
+          [
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
+          ]
+        )
       end
     end
   end
diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb
index 6519d90c..5b12e575 100644
--- a/spec/datadog/ci/span_spec.rb
+++ b/spec/datadog/ci/span_spec.rb
@@ -8,6 +8,94 @@
     end
   end
 
+  describe "#passed?" do
+    context "when status is PASS" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns true" do
+        expect(span.passed?).to eq(true)
+      end
+    end
+
+    context "when status is not PASS" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail")
+      end
+
+      it "returns false" do
+        expect(span.passed?).to eq(false)
+      end
+    end
+  end
+
+  describe "#failed?" do
+    context "when status is FAIL" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail")
+      end
+
+      it "returns true" do
+        expect(span.failed?).to eq(true)
+      end
+    end
+
+    context "when status is not FAIL" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.failed?).to eq(false)
+      end
+    end
+  end
+
+  describe "#skipped?" do
+    context "when status is SKIP" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("skip")
+      end
+
+      it "returns true" do
+        expect(span.skipped?).to eq(true)
+      end
+    end
+
+    context "when status is not SKIP" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.skipped?).to eq(false)
+      end
+    end
+  end
+
+  describe "#undefined?" do
+    context "when status is nil" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return(nil)
+      end
+
+      it "returns true" do
+        expect(span.undefined?).to eq(true)
+      end
+    end
+
+    context "when status is not nil" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.undefined?).to eq(false)
+      end
+    end
+  end
+
   describe "#passed!" do
     it "sets the status to PASS" do
       expect(tracer_span).to receive(:set_tag).with("test.status", "pass")
diff --git a/spec/datadog/ci/test_spec.rb b/spec/datadog/ci/test_spec.rb
index 874127a5..75d5d642 100644
--- a/spec/datadog/ci/test_spec.rb
+++ b/spec/datadog/ci/test_spec.rb
@@ -47,6 +47,26 @@
     it { is_expected.to eq("test suite name") }
   end
 
+  describe "#test_suite" do
+    subject(:test_suite) { ci_test.test_suite }
+    let(:ci_test) { described_class.new(tracer_span) }
+
+    context "when test suite name is set" do
+      before do
+        allow(ci_test).to receive(:test_suite_name).and_return("test suite name")
+        allow(Datadog::CI).to receive(:active_test_suite).with("test suite name").and_return("test suite")
+      end
+
+      it { is_expected.to eq("test suite") }
+    end
+
+    context "when test suite name is not set" do
+      before { allow(ci_test).to receive(:test_suite_name).and_return(nil) }
+
+      it { is_expected.to be_nil }
+    end
+  end
+
   describe "#test_module_id" do
     subject(:test_module_id) { ci_test.test_module_id }
     let(:ci_test) { described_class.new(tracer_span) }

From a18f013d15e8550b46421dba7f1951df513167a7 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 29 Dec 2023 18:32:31 +0100
Subject: [PATCH 08/11] increase buffer size for tracer's writer by a factor of
 10 to accomodate faster test suites

---
 lib/datadog/ci/configuration/components.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb
index 5b8b4686..d322e30d 100644
--- a/lib/datadog/ci/configuration/components.rb
+++ b/lib/datadog/ci/configuration/components.rb
@@ -63,7 +63,8 @@ def activate_ci!(settings)
           writer_options = settings.ci.writer_options
           if test_visibility_transport
             writer_options[:transport] = test_visibility_transport
-            writer_options[:shutdown_timeout] = 60
+            writer_options[:shutdown_timeout] = 300
+            writer_options[:buffer_size] = 10_000
 
             settings.tracing.test_mode.async = true
           end

From d7013d67d8691007223fa04a077cdd387078cafc Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Fri, 29 Dec 2023 18:34:58 +0100
Subject: [PATCH 09/11] revert shutdown_timeout setting

---
 lib/datadog/ci/configuration/components.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb
index d322e30d..bb951d45 100644
--- a/lib/datadog/ci/configuration/components.rb
+++ b/lib/datadog/ci/configuration/components.rb
@@ -63,7 +63,7 @@ def activate_ci!(settings)
           writer_options = settings.ci.writer_options
           if test_visibility_transport
             writer_options[:transport] = test_visibility_transport
-            writer_options[:shutdown_timeout] = 300
+            writer_options[:shutdown_timeout] = 60
             writer_options[:buffer_size] = 10_000
 
             settings.tracing.test_mode.async = true

From 1c278b7bf4a0f98221bec63c0950297e9a57c934 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Tue, 2 Jan 2024 12:33:21 +0100
Subject: [PATCH 10/11] change name for concurrent test suites to make it
 shorter

---
 lib/datadog/ci/contrib/minitest/hooks.rb                 | 2 +-
 spec/datadog/ci/contrib/minitest/instrumentation_spec.rb | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 0d560e92..00a3a3bd 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -18,7 +18,7 @@ def before_setup
 
             test_suite_name = Suite.name(self.class, name)
             if parallel?
-              test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
+              test_suite_name = "#{test_suite_name} (#{name} concurrently)"
 
               # for parallel execution we need to start a new test suite for each test
               CI.start_test_suite(test_suite_name)
diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
index bca20dd3..8444a110 100644
--- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb
@@ -651,10 +651,10 @@ def test_b_2
         test_suite_names = test_suite_spans.map { |span| span.name }.sort
         expect(test_suite_names).to eq(
           [
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_a_1 concurrently)",
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_a_2 concurrently)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_b_1 concurrently)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_b_2 concurrently)"
           ]
         )
       end

From 452b59df7e8f6d188c9696de6347e4b6f8281fdc Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Wed, 3 Jan 2024 08:43:46 +0100
Subject: [PATCH 11/11] as ActiveSupport::TestCase overrides test_order, use
 presence of Minitest::Parallel::Test in ancestors chain to detect
 parallelized Runnable

---
 .../ci/contrib/minitest/{suite.rb => helpers.rb}    | 10 ++++++----
 lib/datadog/ci/contrib/minitest/hooks.rb            | 12 ++++--------
 lib/datadog/ci/contrib/minitest/runnable.rb         | 10 +++-------
 sig/datadog/ci/contrib/minitest/helpers.rbs         | 13 +++++++++++++
 sig/datadog/ci/contrib/minitest/hooks.rbs           |  2 --
 sig/datadog/ci/contrib/minitest/runnable.rbs        |  2 --
 sig/datadog/ci/contrib/minitest/suite.rbs           | 11 -----------
 7 files changed, 26 insertions(+), 34 deletions(-)
 rename lib/datadog/ci/contrib/minitest/{suite.rb => helpers.rb} (66%)
 create mode 100644 sig/datadog/ci/contrib/minitest/helpers.rbs
 delete mode 100644 sig/datadog/ci/contrib/minitest/suite.rbs

diff --git a/lib/datadog/ci/contrib/minitest/suite.rb b/lib/datadog/ci/contrib/minitest/helpers.rb
similarity index 66%
rename from lib/datadog/ci/contrib/minitest/suite.rb
rename to lib/datadog/ci/contrib/minitest/helpers.rb
index c675384f..2837a3a5 100644
--- a/lib/datadog/ci/contrib/minitest/suite.rb
+++ b/lib/datadog/ci/contrib/minitest/helpers.rb
@@ -4,15 +4,17 @@ module Datadog
   module CI
     module Contrib
       module Minitest
-        # Minitest integration constants
-        # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
-        module Suite
-          def self.name(klass, method_name)
+        module Helpers
+          def self.test_suite_name(klass, method_name)
             source_location, = klass.instance_method(method_name).source_location
             source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
 
             "#{klass.name} at #{source_file_path}"
           end
+
+          def self.parallel?(klass)
+            klass.ancestors.include?(::Minitest::Parallel::Test)
+          end
         end
       end
     end
diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
index 00a3a3bd..bef8591a 100644
--- a/lib/datadog/ci/contrib/minitest/hooks.rb
+++ b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -2,7 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
-require_relative "suite"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -16,8 +16,8 @@ def before_setup
 
             test_name = "#{class_name}##{name}"
 
-            test_suite_name = Suite.name(self.class, name)
-            if parallel?
+            test_suite_name = Helpers.test_suite_name(self.class, name)
+            if Helpers.parallel?(self.class)
               test_suite_name = "#{test_suite_name} (#{name} concurrently)"
 
               # for parallel execution we need to start a new test suite for each test
@@ -41,7 +41,7 @@ def after_teardown
             return super unless test_span
 
             finish_test(test_span, result_code)
-            if parallel?
+            if Helpers.parallel?(self.class)
               finish_test_suite(test_span.test_suite, result_code)
             end
 
@@ -78,10 +78,6 @@ def finish_with_result(span, result_code)
             span.finish
           end
 
-          def parallel?
-            self.class.test_order == :parallel
-          end
-
           def datadog_configuration
             Datadog.configuration.ci[:minitest]
           end
diff --git a/lib/datadog/ci/contrib/minitest/runnable.rb b/lib/datadog/ci/contrib/minitest/runnable.rb
index f1903de0..cd239ae0 100644
--- a/lib/datadog/ci/contrib/minitest/runnable.rb
+++ b/lib/datadog/ci/contrib/minitest/runnable.rb
@@ -1,4 +1,4 @@
-require_relative "suite"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -12,12 +12,12 @@ def self.included(base)
           module ClassMethods
             def run(*)
               return super unless datadog_configuration[:enabled]
-              return super if parallel?
+              return super if Helpers.parallel?(self)
 
               method = runnable_methods.first
               return super if method.nil?
 
-              test_suite_name = Suite.name(self, method)
+              test_suite_name = Helpers.test_suite_name(self, method)
 
               test_suite = Datadog::CI.start_test_suite(test_suite_name)
               test_suite.passed! # will be overridden if any test fails
@@ -31,10 +31,6 @@ def run(*)
 
             private
 
-            def parallel?
-              test_order == :parallel
-            end
-
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
diff --git a/sig/datadog/ci/contrib/minitest/helpers.rbs b/sig/datadog/ci/contrib/minitest/helpers.rbs
new file mode 100644
index 00000000..4ceb76c5
--- /dev/null
+++ b/sig/datadog/ci/contrib/minitest/helpers.rbs
@@ -0,0 +1,13 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Helpers
+          def self.test_suite_name: (untyped klass, String? method_name) -> ::String
+
+          def self.parallel?: (untyped klass) -> bool
+        end
+      end
+    end
+  end
+end
diff --git a/sig/datadog/ci/contrib/minitest/hooks.rbs b/sig/datadog/ci/contrib/minitest/hooks.rbs
index ccc07f7f..4cad85e4 100644
--- a/sig/datadog/ci/contrib/minitest/hooks.rbs
+++ b/sig/datadog/ci/contrib/minitest/hooks.rbs
@@ -11,8 +11,6 @@ module Datadog
 
           private
 
-          def parallel?: () -> bool
-
           def datadog_configuration: () -> untyped
 
           def finish_test: (Datadog::CI::Test test_span, String result_code) -> void
diff --git a/sig/datadog/ci/contrib/minitest/runnable.rbs b/sig/datadog/ci/contrib/minitest/runnable.rbs
index 09b724f6..a9e11f58 100644
--- a/sig/datadog/ci/contrib/minitest/runnable.rbs
+++ b/sig/datadog/ci/contrib/minitest/runnable.rbs
@@ -11,8 +11,6 @@ module Datadog
 
             private
 
-            def parallel?: () -> bool
-
             def datadog_configuration: () -> untyped
 
             def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
diff --git a/sig/datadog/ci/contrib/minitest/suite.rbs b/sig/datadog/ci/contrib/minitest/suite.rbs
deleted file mode 100644
index e87a538a..00000000
--- a/sig/datadog/ci/contrib/minitest/suite.rbs
+++ /dev/null
@@ -1,11 +0,0 @@
-module Datadog
-  module CI
-    module Contrib
-      module Minitest
-        module Suite
-          def self.name: (untyped klass, String? method_name) -> ::String
-        end
-      end
-    end
-  end
-end