Merge pull request #77 from DataDog/anmarchenko/test_suites_support

[CIVIS-2844] Test suites support
DataDog · Dec 5, 2023 · cc49b3c · cc49b3c
2 parents c42591d + bf6bea8
commit cc49b3c
Show file tree

Hide file tree

Showing 41 changed files with 1,061 additions and 239 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ Gemfile-*.lock
 .envrc
 .ruby-version
 .DS_Store
+/test.rb
diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb
@@ -11,14 +11,16 @@ module Datadog
   module CI
     class << self
       # Starts a {Datadog::CI::TestSesstion ci_test_session} that represents the whole test session run.
+      #
       # Read Datadog documentation on test sessions
       # [here](https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#sessions).
-      # Raises an error if a session is already active.
+      #
+      # Returns the existing test session if one is already active. There is at most a single test session per process.
       #
       # The {#start_test_session} method is used to mark the start of the test session:
       # ```
       # Datadog::CI.start_test_session(
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
       #
@@ -28,14 +30,15 @@ class << self
       #
       # Remember that calling {Datadog::CI::TestSession#finish} is mandatory.
       #
-      # @param [String] service_name the service name for this session
+      # @param [String] service_name the service name for this session (optional, defaults to DD_SERVICE)
       # @param [Hash<String,String>] tags extra tags which should be added to the test session.
       # @return [Datadog::CI::TestSession] returns the active, running {Datadog::CI::TestSession}.
       # @return [Datadog::CI::NullSpan] ci_span null object if CI visibility is disabled or if old Datadog agent is
       #         detected and test suite level visibility cannot be supported.
       #
       # @public_api
       def start_test_session(service_name: nil, tags: {})
+        service_name ||= Datadog.configuration.service
         recorder.start_test_session(service_name: service_name, tags: tags)
       end
 
@@ -46,7 +49,7 @@ def start_test_session(service_name: nil, tags: {})
       # ```
       # # start a test session
       # Datadog::CI.start_test_session(
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
       #
@@ -67,14 +70,14 @@ def active_test_session
       # Read Datadog documentation on test modules:
       # [here](https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#module).
       #
-      # Raises an error if a module is already active. There is at most a single test module per process
+      # Returns the existing test session if one is already active. There is at most a single test module per process
       # active at any given time.
       #
       # The {#start_test_module} method is used to mark the start of the test session:
       # ```
       # Datadog::CI.start_test_module(
       #   "my-module",
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
       #
@@ -85,8 +88,8 @@ def active_test_session
       # Remember that calling {Datadog::CI::TestModule#finish} is mandatory.
       #
       # @param [String] test_module_name the name for this module
-      # @param [String] service_name the service name for this session
-      # @param [Hash<String,String>] tags extra tags which should be added to the test module.
+      # @param [String] service_name the service name for this session (optional, inherited from test session if not provided)
+      # @param [Hash<String,String>] tags extra tags which should be added to the test module (optional, some tags are inherited from test session).
       # @return [Datadog::CI::TestModule] returns the active, running {Datadog::CI::TestModule}.
       # @return [Datadog::CI::NullSpan] ci_span null object if CI visibility is disabled or if old Datadog agent is
       #         detected and test suite level visibility cannot be supported.
@@ -104,7 +107,7 @@ def start_test_module(test_module_name, service_name: nil, tags: {})
       # # start a test module
       # Datadog::CI.start_test_module(
       #   "my-module",
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
       #
@@ -119,6 +122,61 @@ def active_test_module
         recorder.active_test_module
       end
 
+      # Starts a {Datadog::CI::TestSuite ci_test_suite} that represents a single test suite.
+      # If a test suite with given name is running, returns the existing test suite.
+      #
+      # Read Datadog documentation on test suites:
+      # [here](https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#module).
+      #
+      # The {#start_test_suite} method is used to mark the start of a test suite:
+      # ```
+      # Datadog::CI.start_test_suite(
+      #   "calculator_tests",
+      #   service_name: "my-web-site-tests",
+      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
+      # )
+      #
+      # # Somewhere else after the suite has ended
+      # Datadog::CI.active_test_suite("calculator_tests").finish
+      # ```
+      #
+      # Remember that calling {Datadog::CI::TestSuite#finish} is mandatory.
+      #
+      # @param [String] test_suite_name the name of the test suite
+      # @param [String] service_name the service name for this test suite (optional, inherited from test session if not provided)
+      # @param [Hash<String,String>] tags extra tags which should be added to the test module (optional, some tags are inherited from test session)
+      # @return [Datadog::CI::TestSuite] returns the active, running {Datadog::CI::TestSuite}.
+      # @return [Datadog::CI::NullSpan] ci_span null object if CI visibility is disabled or if old Datadog agent is
+      #         detected and test suite level visibility cannot be supported.
+      #
+      # @public_api
+      def start_test_suite(test_suite_name, service_name: nil, tags: {})
+        recorder.start_test_suite(test_suite_name, service_name: service_name, tags: tags)
+      end
+
+      # The active, unfinished test suite.
+      #
+      # Usage:
+      #
+      # ```
+      # # start a test suite
+      # Datadog::CI.start_test_suite(
+      #   "calculator_tests",
+      #   service_name: "my-web-site-tests",
+      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
+      # )
+      #
+      # # Somewhere else after the suite has ended
+      # test_suite = Datadog::CI.active_test_suite("calculator_tests")
+      # test_suite.finish
+      # ```
+      #
+      # @return [Datadog::CI::TestSuite] the active test suite
+      # @return [nil] if no test suite with given name is active
+      def active_test_suite(test_suite_name)
+        recorder.active_test_suite(test_suite_name)
+      end
+
       # Return a {Datadog::CI::Test ci_test} that will trace a test called `test_name`.
       # Raises an error if a test is already active.
       # If there is an active test session, the new test will be connected to the session.
@@ -148,7 +206,7 @@ def active_test_module
       # ```
       # ci_test = Datadog::CI.trace_test(
       #   "test_add_two_numbers',
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   operation_name: "test",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
@@ -159,6 +217,7 @@ def active_test_module
       # Remember that in this case, calling {Datadog::CI::Test#finish} is mandatory.
       #
       # @param [String] test_name {Datadog::CI::Test} name (example: "test_add_two_numbers").
+      # @param [String] test_suite_name name of test suite this test belongs to (example: "CalculatorTest").
       # @param [String] operation_name defines label for a test span in trace view ("test" if it's missing)
       # @param [String] service_name the service name for this test
       # @param [Hash<String,String>] tags extra tags which should be added to the test.
@@ -171,8 +230,8 @@ def active_test_module
       # @yieldparam [Datadog::CI::NullSpan] ci_span null object if CI visibility is disabled
       #
       # @public_api
-      def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, &block)
-        recorder.trace_test(test_name, service_name: service_name, operation_name: operation_name, tags: tags, &block)
+      def trace_test(test_name, test_suite_name, service_name: nil, operation_name: "test", tags: {}, &block)
+        recorder.trace_test(test_name, test_suite_name, service_name: service_name, operation_name: operation_name, tags: tags, &block)
       end
 
       # Same as {#trace_test} but it does not accept a block.
@@ -183,7 +242,7 @@ def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, &
       # ```
       # ci_test = Datadog::CI.start_test(
       #   "test_add_two_numbers',
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   operation_name: "test",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
@@ -192,15 +251,16 @@ def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, &
       # ```
       #
       # @param [String] test_name {Datadog::CI::Test} name (example: "test_add_two_numbers").
+      # @param [String] test_suite_name name of test suite this test belongs to (example: "CalculatorTest").
       # @param [String] operation_name the resource this span refers, or `test` if it's missing
       # @param [String] service_name the service name for this span.
       # @param [Hash<String,String>] tags extra tags which should be added to the test.
       # @return [Datadog::CI::Test] Returns the active, unfinished {Datadog::CI::Test}.
       # @return [Datadog::CI::NullSpan] ci_span null object if CI visibility is disabled
       #
       # @public_api
-      def start_test(test_name, service_name: nil, operation_name: "test", tags: {})
-        recorder.trace_test(test_name, service_name: service_name, operation_name: operation_name, tags: tags)
+      def start_test(test_name, test_suite_name, service_name: nil, operation_name: "test", tags: {})
+        recorder.trace_test(test_name, test_suite_name, service_name: service_name, operation_name: operation_name, tags: tags)
       end
 
       # Trace any custom span inside a test. For example, you could trace:
@@ -284,7 +344,7 @@ def active_span(span_type)
       # # start a test
       # Datadog::CI.start_test(
       #   "test_add_two_numbers',
-      #   service: "my-web-site-tests",
+      #   service_name: "my-web-site-tests",
       #   operation_name: "test",
       #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
       # )
@@ -316,6 +376,11 @@ def deactivate_test_module
         recorder.deactivate_test_module
       end
 
+      # Internal only, to finish a test suite use Datadog::CI::TestSuite#finish
+      def deactivate_test_suite(test_suite_name)
+        recorder.deactivate_test_suite(test_suite_name)
+      end
+
       private
 
       def components

diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb
@@ -62,7 +62,7 @@ def self.add_settings!(base)
                 fetch_integration(integration_name).configuration
               end
 
-              # TODO: Deprecate in the next major version, as `instrument` better describes this method's purpose
+              # @deprecated Will be removed on datadog-ci-rb 1.0.
               alias_method :use, :instrument
 
               option :trace_flush

diff --git a/lib/datadog/ci/context/global.rb b/lib/datadog/ci/context/global.rb
@@ -6,9 +6,30 @@ module Context
       # This context is shared between threads and represents the current test session and test module.
       class Global
         def initialize
-          @mutex = Mutex.new
+          # we are using Monitor instead of Mutex because it is reentrant
+          @mutex = Monitor.new
+
           @test_session = nil
           @test_module = nil
+          @test_suites = {}
+        end
+
+        def fetch_or_activate_test_suite(test_suite_name, &block)
+          @mutex.synchronize do
+            @test_suites[test_suite_name] ||= block.call
+          end
+        end
+
+        def fetch_or_activate_test_module(&block)
+          @mutex.synchronize do
+            @test_module ||= block.call
+          end
+        end
+
+        def fetch_or_activate_test_session(&block)
+          @mutex.synchronize do
+            @test_session ||= block.call
+          end
         end
 
         def active_test_module
@@ -19,6 +40,10 @@ def active_test_session
           @test_session
         end
 
+        def active_test_suite(test_suite_name)
+          @mutex.synchronize { @test_suites[test_suite_name] }
+        end
+
         def service
           @mutex.synchronize do
             # thank you RBS for this weirdness
@@ -38,29 +63,17 @@ def inheritable_session_tags
           end
         end
 
-        def activate_test_session!(test_session)
-          @mutex.synchronize do
-            raise "Nested test sessions are not supported. Currently active test session: #{@test_session}" unless @test_session.nil?
-
-            @test_session = test_session
-          end
-        end
-
-        def activate_test_module!(test_module)
-          @mutex.synchronize do
-            raise "Nested test modules are not supported. Currently active test module: #{@test_module}" unless @test_module.nil?
-
-            @test_module = test_module
-          end
-        end
-
         def deactivate_test_session!
           @mutex.synchronize { @test_session = nil }
         end
 
         def deactivate_test_module!
           @mutex.synchronize { @test_module = nil }
         end
+
+        def deactivate_test_suite!(test_suite_name)
+          @mutex.synchronize { @test_suites.delete(test_suite_name) }
+        end
       end
     end
   end

diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb
@@ -29,11 +29,11 @@ def bind_events(config)
           def on_test_case_started(event)
             CI.start_test(
               event.test_case.name,
+              event.test_case.location.file,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE,
-                CI::Ext::Test::TAG_SUITE => event.test_case.location.file
+                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
               },
               service_name: configuration[:service_name],
               operation_name: configuration[:operation_name]

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -20,11 +20,11 @@ def before_setup
 
             CI.start_test(
               test_name,
+              test_suite,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE,
-                CI::Ext::Test::TAG_SUITE => test_suite
+                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
               },
               service_name: configuration[:service_name],
               operation_name: configuration[:operation_name]

diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb
@@ -26,11 +26,11 @@ def run(example_group_instance, reporter)
 
               CI.trace_test(
                 test_name,
+                metadata[:example_group][:file_path],
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE,
-                  CI::Ext::Test::TAG_SUITE => metadata[:example_group][:file_path]
+                  CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
                 },
                 service_name: configuration[:service_name],
                 operation_name: configuration[:operation_name]

diff --git a/lib/datadog/ci/ext/app_types.rb b/lib/datadog/ci/ext/app_types.rb
@@ -7,8 +7,9 @@ module AppTypes
         TYPE_TEST = "test"
         TYPE_TEST_SESSION = "test_session_end"
         TYPE_TEST_MODULE = "test_module_end"
+        TYPE_TEST_SUITE = "test_suite_end"
 
-        CI_SPAN_TYPES = [TYPE_TEST, TYPE_TEST_SESSION, TYPE_TEST_MODULE].freeze
+        CI_SPAN_TYPES = [TYPE_TEST, TYPE_TEST_SESSION, TYPE_TEST_MODULE, TYPE_TEST_SUITE].freeze
       end
     end
   end

diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb
@@ -22,7 +22,8 @@ module Test
         # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
-        SPECIAL_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID].freeze
+        TAG_TEST_SUITE_ID = "_test.suite_id"
+        SPECIAL_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
 
         # tags that can be inherited from the test session
         INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze
@@ -33,7 +34,6 @@ module Test
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 
-        # TODO: is there a better place for SPAN_KIND?
         TAG_SPAN_KIND = "span.kind"
 
         module Status