track test suite per test when executing in parallel mode

lib/datadog/ci/contrib/minitest/hooks.rb

@@ -19,6 +19,9 @@ def before_setup
             test_suite_name = Suite.name(self.class, name)
             if parallel?
               test_suite_name = "#{test_suite_name} (parallel execution of #{test_name})"
+
+              # for parallel execution we need to start a new test suite for each test
+              CI.start_test_suite(test_suite_name)
             end
 
             CI.start_test(
@@ -37,26 +40,44 @@ def after_teardown
             test_span = CI.active_test
             return super unless test_span
 
-            case result_code
-            when "."
-              test_span.passed!
-            when "E", "F"
-              test_suite_name = test_span.test_suite_name
-              test_suite = CI.active_test_suite(test_suite_name) if test_suite_name
-              test_suite.failed! if test_suite
-
-              test_span.failed!(exception: failure)
-            when "S"
-              test_span.skipped!(reason: failure.message)
+            finish_test(test_span, result_code)
+            if parallel?
+              finish_test_suite(test_span.test_suite, result_code)
             end
 
-            test_span.finish
-
             super
           end
 
           private
 
+          def finish_test(test_span, result_code)
+            finish_with_result(test_span, result_code)
+
+            # mark test suite as failed if any test failed
+            if test_span.failed?
+              test_suite = test_span.test_suite
+              test_suite.failed! if test_suite
+            end
+          end
+
+          def finish_test_suite(test_suite, result_code)
+            return unless test_suite
+
+            finish_with_result(test_suite, result_code)
+          end
+
+          def finish_with_result(span, result_code)
+            case result_code
+            when "."
+              span.passed!
+            when "E", "F"
+              span.failed!(exception: failure)
+            when "S"
+              span.skipped!(reason: failure.message)
+            end
+            span.finish
+          end
+
           def parallel?
             self.class.test_order == :parallel
           end

lib/datadog/ci/contrib/minitest/runnable.rb

@@ -13,9 +13,10 @@ module ClassMethods
             def run(*)
               return super unless datadog_configuration[:enabled]
               return super if parallel?
-              return super if runnable_methods.empty?
 
               method = runnable_methods.first
+              return super if method.nil?
+
               test_suite_name = Suite.name(self, method)
 
               test_suite = Datadog::CI.start_test_suite(test_suite_name)

lib/datadog/ci/span.rb

@@ -36,6 +36,30 @@ def span_type
         tracer_span.type
       end
 
+      # Checks whether span status is not set yet.
+      # @return [bool] true if span does not have status
+      def undefined?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS).nil?
+      end
+
+      # Checks whether span status is PASS.
+      # @return [bool] true if span status is PASS
+      def passed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::PASS
+      end
+
+      # Checks whether span status is FAIL.
+      # @return [bool] true if span status is FAIL
+      def failed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::FAIL
+      end
+
+      # Checks whether span status is SKIP.
+      # @return [bool] true if span status is SKIP
+      def skipped?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::SKIP
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!

lib/datadog/ci/test.rb

@@ -21,6 +21,14 @@ def finish
         CI.deactivate_test(self)
       end
 
+      # Running test suite that this test is part of (if any).
+      # @return [Datadog::CI::TestSuite] the test suite this test belongs to
+      # @return [nil] if the test suite is not found
+      def test_suite
+        suite_name = test_suite_name
+        CI.active_test_suite(suite_name) if suite_name
+      end
+
       # Span id of the running test suite this test belongs to.
       # @return [String] the span id of the test suite.
       def test_suite_id

sig/datadog/ci/contrib/minitest/hooks.rbs

@@ -15,6 +15,12 @@ module Datadog
 
           def datadog_configuration: () -> untyped
 
+          def finish_test: (Datadog::CI::Test test_span, String result_code) -> void
+
+          def finish_test_suite: (Datadog::CI::TestSuite? test_suite, String result_code) -> void
+
+          def finish_with_result: (Datadog::CI::Span span, String result_code) -> void
+
           def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha)
         end
       end

sig/datadog/ci/span.rbs

@@ -13,6 +13,14 @@ module Datadog
 
       def service: () -> String
 
+      def undefined?: () -> bool
+
+      def passed?: () -> bool
+
+      def failed?: () -> bool
+
+      def skipped?: () -> bool
+
       def passed!: () -> void
 
       def failed!: (?exception: untyped?) -> void

sig/datadog/ci/test.rbs

@@ -2,6 +2,7 @@ module Datadog
   module CI
     class Test < Span
       def finish: () -> void
+      def test_suite: () -> Datadog::CI::TestSuite?
       def test_suite_id: () -> String?
       def test_suite_name: () -> String?
       def test_module_id: () -> String?

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -569,8 +569,6 @@ class SomeSpec < Minitest::Spec
 
     context "using parallel executor" do
       before(:context) do
-        require "minitest/hell"
-
         Minitest::Runnable.reset
 
         class ParallelTest < Minitest::Test
@@ -602,11 +600,14 @@ def test_b_2
         end
       end
 
-      it "traces all tests correctly" do
-        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
-        test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort
+      it "traces all tests correctly, assigning a separate test suite to each of them" do
         test_threads = test_spans.map { |span| span.get_tag("minitest_thread") }.uniq
 
+        # make sure that tests were executed concurrently
+        # note that this test could be flaky
+        expect(test_threads.count).to be > 1
+
+        test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
         expect(test_names).to eq(
           [
             "TestA#test_a_1",
@@ -616,18 +617,8 @@ def test_b_2
           ]
         )
 
-        expect(test_suite_names).to eq(
-          [
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
-            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
-            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
-          ]
-        )
-
-        # make sure that tests were executed concurrently
-        # note that this test could be flaky
-        expect(test_threads.count).to be > 1
+        test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq
+        expect(test_suite_ids).to have(4).items
       end
 
       it "connects tests to a single test session and a single test module" do
@@ -648,16 +639,24 @@ def test_b_2
         # with parallel execution test durations sum should be greater than test session duration
         expect(test_durations_sum).to be > test_session_duration
 
-        # but each individual test duration should be less than test session duration
+        # each individual test duration should be less than test session duration
         test_spans.each do |span|
           expect(span.duration).to be < test_session_duration
         end
       end
 
       it "creates test suite spans" do
-        skip("test suite spans for parallel execution pending")
-
         expect(test_suite_spans).to have(4).items
+
+        test_suite_names = test_suite_spans.map { |span| span.name }.sort
+        expect(test_suite_names).to eq(
+          [
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_1)",
+            "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestA#test_a_2)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_1)",
+            "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (parallel execution of TestB#test_b_2)"
+          ]
+        )
       end
     end
   end

spec/datadog/ci/span_spec.rb

@@ -8,6 +8,94 @@
     end
   end
 
+  describe "#passed?" do
+    context "when status is PASS" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns true" do
+        expect(span.passed?).to eq(true)
+      end
+    end
+
+    context "when status is not PASS" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail")
+      end
+
+      it "returns false" do
+        expect(span.passed?).to eq(false)
+      end
+    end
+  end
+
+  describe "#failed?" do
+    context "when status is FAIL" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail")
+      end
+
+      it "returns true" do
+        expect(span.failed?).to eq(true)
+      end
+    end
+
+    context "when status is not FAIL" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.failed?).to eq(false)
+      end
+    end
+  end
+
+  describe "#skipped?" do
+    context "when status is SKIP" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("skip")
+      end
+
+      it "returns true" do
+        expect(span.skipped?).to eq(true)
+      end
+    end
+
+    context "when status is not SKIP" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.skipped?).to eq(false)
+      end
+    end
+  end
+
+  describe "#undefined?" do
+    context "when status is nil" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return(nil)
+      end
+
+      it "returns true" do
+        expect(span.undefined?).to eq(true)
+      end
+    end
+
+    context "when status is not nil" do
+      before do
+        allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass")
+      end
+
+      it "returns false" do
+        expect(span.undefined?).to eq(false)
+      end
+    end
+  end
+
   describe "#passed!" do
     it "sets the status to PASS" do
       expect(tracer_span).to receive(:set_tag).with("test.status", "pass")

spec/datadog/ci/test_spec.rb

@@ -47,6 +47,26 @@
     it { is_expected.to eq("test suite name") }
   end
 
+  describe "#test_suite" do
+    subject(:test_suite) { ci_test.test_suite }
+    let(:ci_test) { described_class.new(tracer_span) }
+
+    context "when test suite name is set" do
+      before do
+        allow(ci_test).to receive(:test_suite_name).and_return("test suite name")
+        allow(Datadog::CI).to receive(:active_test_suite).with("test suite name").and_return("test suite")
+      end
+
+      it { is_expected.to eq("test suite") }
+    end
+
+    context "when test suite name is not set" do
+      before { allow(ci_test).to receive(:test_suite_name).and_return(nil) }
+
+      it { is_expected.to be_nil }
+    end
+  end
+
   describe "#test_module_id" do
     subject(:test_module_id) { ci_test.test_module_id }
     let(:ci_test) { described_class.new(tracer_span) }