instrument RSpec::Core::ExampleGroup to trace test suites

DataDog · Dec 8, 2023 · 3e02daa · 3e02daa
1 parent 8062258
commit 3e02daa
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 28 deletions.
diff --git a/lib/datadog/ci/contrib/rspec/patcher.rb b/lib/datadog/ci/contrib/rspec/patcher.rb
@@ -21,6 +21,26 @@ def target_version
           def patch
             ::RSpec::Core::Example.include(Example)
             ::RSpec::Core::Runner.include(Runner)
+
+            ::RSpec::Core::ExampleGroup.class_eval do
+              class << self
+                alias_method :__run, :run
+
+                def run(reporter = ::RSpec::Core::NullReporter)
+                  return __run(reporter) unless top_level?
+
+                  test_suite = Datadog::CI.start_test_suite(file_path)
+                  result = __run(reporter)
+                  if result
+                    test_suite.passed!
+                  else
+                    test_suite.failed!
+                  end
+                  test_suite.finish
+                  result
+                end
+              end
+            end
           end
         end
       end

diff --git a/sig/datadog/ci.rbs b/sig/datadog/ci.rbs
@@ -18,6 +18,8 @@ module Datadog
 
     def self.active_test: () -> Datadog::CI::Test?
 
+    def self.active_test_suite: (String test_suite_name) -> Datadog::CI::TestSuite?
+
     def self.active_span: (String span_type) -> Datadog::CI::Span?
 
     def self.deactivate_test: (Datadog::CI::Test test) -> void

diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb
@@ -29,19 +29,19 @@ def with_new_rspec_environment
       end.tap(&:run)
     end
 
-    expect(span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-    expect(span.name).to eq("some test foo")
-    expect(span.resource).to eq("some test foo")
-    expect(span.service).to eq("lspec")
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some test foo")
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK)
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+    expect(first_test_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
+    expect(first_test_span.name).to eq("some test foo")
+    expect(first_test_span.resource).to eq("some test foo")
+    expect(first_test_span.service).to eq("lspec")
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some test foo")
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
       Datadog::CI::Contrib::RSpec::Integration.version.to_s
     )
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS)
   end
 
   it "creates correct span on shared examples" do
@@ -52,7 +52,7 @@ def with_new_rspec_environment
       end.tap(&:run)
     end
 
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
   end
 
   it "creates spans for several examples" do
@@ -69,7 +69,7 @@ def with_new_rspec_environment
       end.run
     end
 
-    expect(spans).to have(num_examples).items
+    expect(test_spans).to have(num_examples).items
   end
 
   it "creates span for unnamed examples" do
@@ -79,7 +79,7 @@ def with_new_rspec_environment
       end.run
     end
 
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to match(/some unnamed test example at .+/)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to match(/some unnamed test example at .+/)
   end
 
   it "creates span for deeply nested examples" do
@@ -111,9 +111,9 @@ def with_new_rspec_environment
       end.tap(&:run)
     end
 
-    expect(span.resource).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo")
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo")
-    expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
+    expect(first_test_span.resource).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo")
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo")
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path)
   end
 
   it "creates spans for example with instrumentation" do
@@ -127,21 +127,22 @@ def with_new_rspec_environment
       end.tap(&:run)
     end
 
-    expect(spans).to have(2).items
+    expect(test_spans).to have(1).items
+    expect(tracer_spans).to have(1).items
 
-    spans.each do |span|
+    tracer_spans.each do |span|
       expect(span.get_tag(Datadog::Tracing::Metadata::Ext::Distributed::TAG_ORIGIN))
         .to eq(Datadog::CI::Ext::Test::CONTEXT_ORIGIN)
     end
   end
 
   context "catches failures" do
     def expect_failure
-      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::FAIL)
-      expect(span).to have_error
-      expect(span).to have_error_type
-      expect(span).to have_error_message
-      expect(span).to have_error_stack
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::FAIL)
+      expect(first_test_span).to have_error
+      expect(first_test_span).to have_error_type
+      expect(first_test_span).to have_error_message
+      expect(first_test_span).to have_error_stack
     end
 
     it "within let" do
@@ -215,7 +216,7 @@ def devnull
 
     def rspec_session_run(with_failed_test: false)
       with_new_rspec_environment do
-        RSpec.describe "SomeTest" do
+        spec = RSpec.describe "SomeTest" do
           it "foo" do
             # DO NOTHING
           end
@@ -229,6 +230,8 @@ def rspec_session_run(with_failed_test: false)
 
         options = ::RSpec::Core::ConfigurationOptions.new(%w[--pattern none])
         ::RSpec::Core::Runner.new(options).run(devnull, devnull)
+
+        spec
       end
     end
 
@@ -280,6 +283,39 @@ def rspec_session_run(with_failed_test: false)
       )
     end
 
+    it "creates test suite span" do
+      spec = rspec_session_run
+
+      expect(test_suite_span).not_to be_nil
+
+      expect(test_suite_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SUITE)
+      expect(test_suite_span.name).to eq(spec.file_path)
+
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
+        Datadog::CI::Ext::AppTypes::TYPE_TEST
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
+        Datadog::CI::Ext::Test::TEST_TYPE
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
+        Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
+        Datadog::CI::Contrib::RSpec::Integration.version.to_s
+      )
+      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::PASS
+      )
+    end
+
+    it "connects test to the session, module, and suite" do
+      rspec_session_run
+
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s)
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s)
+      expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s)
+    end
+
     context "with failures" do
       it "creates test session span with failed state" do
         rspec_session_run(with_failed_test: true)
@@ -298,6 +334,15 @@ def rspec_session_run(with_failed_test: false)
           Datadog::CI::Ext::Test::Status::FAIL
         )
       end
+
+      it "creates test suite span with failed state" do
+        rspec_session_run(with_failed_test: true)
+
+        expect(test_suite_span).not_to be_nil
+        expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::FAIL
+        )
+      end
     end
   end
 end
diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb
@@ -108,11 +108,19 @@ def test_suite_span
   end
 
   def first_test_span
-    spans.find { |span| span.type == "test" }
+    test_spans.first
   end
 
   def first_other_span
-    spans.find { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) }
+    tracer_spans.first
+  end
+
+  def test_spans
+    spans.filter { |span| span.type == "test" }
+  end
+
+  def tracer_spans
+    spans.filter { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) }
   end
 
   # Returns traces and caches it (similar to +let(:traces)+).

diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs
@@ -14,4 +14,8 @@ end
 
 module RSpec::Core::Runner
   def run: (untyped err, untyped out) -> untyped
-end
+end
+
+module RSpec::Core::ExampleGroup
+  def self.run: () -> bool
+end