Merge pull request #202 from DataDog/anmarchenko/telemetry_metrics_ma…

…nual_api

[SDTEST-116] Implement manual_api_events metric

lib/datadog/ci.rb

@@ -2,7 +2,9 @@
 
 require_relative "ci/version"
 require_relative "ci/utils/configuration"
+require_relative "ci/utils/telemetry"
 require_relative "ci/ext/app_types"
+require_relative "ci/ext/telemetry"
 
 require "datadog/core"
 
@@ -39,6 +41,11 @@ class << self
       # @return [Datadog::CI::TestSession] the active, running {Datadog::CI::TestSession}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
       def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {})
+        Utils::Telemetry.inc(
+          Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
+          1,
+          {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::SESSION}
+        )
         test_visibility.start_test_session(service: service, tags: tags)
       end
 
@@ -93,6 +100,12 @@ def active_test_session
       # @return [Datadog::CI::TestModule] the active, running {Datadog::CI::TestModule}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
       def start_test_module(test_module_name, service: nil, tags: {})
+        Utils::Telemetry.inc(
+          Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
+          1,
+          {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::MODULE}
+        )
+
         test_visibility.start_test_module(test_module_name, service: service, tags: tags)
       end
 
@@ -145,6 +158,12 @@ def active_test_module
       # @return [Datadog::CI::TestSuite] the active, running {Datadog::CI::TestSuite}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
       def start_test_suite(test_suite_name, service: nil, tags: {})
+        Utils::Telemetry.inc(
+          Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
+          1,
+          {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::SUITE}
+        )
+
         test_visibility.start_test_suite(test_suite_name, service: service, tags: tags)
       end
 
@@ -222,6 +241,12 @@ def active_test_suite(test_suite_name)
       # @yieldparam [Datadog::CI::Test] ci_test the newly created and active [Datadog::CI::Test]
       # @yieldparam [nil] if CI mode is disabled
       def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+        Utils::Telemetry.inc(
+          Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
+          1,
+          {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::TEST}
+        )
+
         test_visibility.trace_test(test_name, test_suite_name, service: service, tags: tags, &block)
       end
 
@@ -248,6 +273,11 @@ def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
       # @return [Datadog::CI::Test] the active, unfinished {Datadog::CI::Test}.
       # @return [nil] if CI mode is disabled.
       def start_test(test_name, test_suite_name, service: nil, tags: {})
+        Utils::Telemetry.inc(
+          Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
+          1,
+          {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::TEST}
+        )
         test_visibility.trace_test(test_name, test_suite_name, service: service, tags: tags)
       end
 

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -35,14 +35,14 @@ def bind_events(config)
           end
 
           def on_test_run_started(event)
-            CI.start_test_session(
+            test_visibility_component.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
               },
               service: configuration[:service_name]
             )
-            CI.start_test_module(Ext::FRAMEWORK)
+            test_visibility_component.start_test_module(Ext::FRAMEWORK)
           end
 
           def on_test_run_finished(event)
@@ -70,7 +70,7 @@ def on_test_case_started(event)
 
             start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
 
-            test_span = CI.start_test(
+            test_span = test_visibility_component.trace_test(
               event.test_case.name,
               test_suite_name,
               tags: tags,
@@ -82,19 +82,19 @@ def on_test_case_started(event)
           end
 
           def on_test_case_finished(event)
-            test_span = CI.active_test
+            test_span = test_visibility_component.active_test
             return if test_span.nil?
 
             finish_span(test_span, event.result)
             @failed_tests_count += 1 if test_span.failed?
           end
 
           def on_test_step_started(event)
-            CI.trace(event.test_step.to_s, type: Ext::STEP_SPAN_TYPE)
+            test_visibility_component.trace(event.test_step.to_s, type: Ext::STEP_SPAN_TYPE)
           end
 
           def on_test_step_finished(event)
-            current_step_span = CI.active_span
+            current_step_span = test_visibility_component.active_span
             return if current_step_span.nil?
 
             finish_span(current_step_span, event.result)
@@ -130,8 +130,8 @@ def finish_span(span, result)
           def finish_session(result)
             finish_current_test_suite
 
-            test_session = CI.active_test_session
-            test_module = CI.active_test_module
+            test_session = test_visibility_component.active_test_session
+            test_module = test_visibility_component.active_test_module
 
             return unless test_session && test_module
 
@@ -150,7 +150,7 @@ def finish_session(result)
           def start_test_suite(test_suite_name)
             finish_current_test_suite
 
-            @current_test_suite = CI.start_test_suite(test_suite_name)
+            @current_test_suite = test_visibility_component.start_test_suite(test_suite_name)
           end
 
           def finish_current_test_suite
@@ -197,6 +197,10 @@ def ok?(result, strict)
           def configuration
             Datadog.configuration.ci[:cucumber]
           end
+
+          def test_visibility_component
+            Datadog.send(:components).test_visibility
+          end
         end
       end
     end

lib/datadog/ci/contrib/minitest/runnable.rb

@@ -19,7 +19,7 @@ def run(*args)
 
               test_suite_name = Helpers.test_suite_name(self, method)
 
-              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+              test_suite = test_visibility_component.start_test_suite(test_suite_name)
 
               results = super
               return results unless test_suite
@@ -34,6 +34,10 @@ def run(*args)
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

lib/datadog/ci/contrib/minitest/runner.rb

@@ -18,21 +18,25 @@ def init_plugins(*args)
 
               return unless datadog_configuration[:enabled]
 
-              CI.start_test_session(
+              test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )
-              CI.start_test_module(Ext::FRAMEWORK)
+              test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end
 
             private
 
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

lib/datadog/ci/contrib/minitest/test.rb

@@ -26,12 +26,12 @@ def before_setup
                 test_suite_name = "#{test_suite_name} (#{name} concurrently)"
 
                 # for parallel execution we need to start a new test suite for each test
-                CI.start_test_suite(test_suite_name)
+                test_visibility_component.start_test_suite(test_suite_name)
               end
 
               source_file, line_number = method(name).source_location
 
-              test_span = CI.start_test(
+              test_span = test_visibility_component.trace_test(
                 name,
                 test_suite_name,
                 tags: {
@@ -47,7 +47,7 @@ def before_setup
             end
 
             def after_teardown
-              test_span = CI.active_test
+              test_span = test_visibility_component.active_test
               return super unless test_span
 
               finish_with_result(test_span, result_code)
@@ -77,6 +77,10 @@ def finish_with_result(span, result_code)
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
 
           module ClassMethods

lib/datadog/ci/contrib/rspec/example.rb

@@ -34,10 +34,10 @@ def run(*args)
 
               if ci_queue?
                 suite_name = "#{suite_name} (ci-queue running example [#{test_name}])"
-                test_suite_span = CI.start_test_suite(suite_name)
+                test_suite_span = test_visibility_component.start_test_suite(suite_name)
               end
 
-              CI.trace_test(
+              test_visibility_component.trace_test(
                 test_name,
                 suite_name,
                 tags: {
@@ -99,6 +99,10 @@ def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
 
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
+
             def ci_queue?
               !!defined?(::RSpec::Queue::ExampleExtension) &&
                 self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -21,7 +21,7 @@ def run(reporter = ::RSpec::Core::NullReporter)
               return super unless top_level?
 
               suite_name = "#{description} at #{file_path}"
-              test_suite = Datadog::CI.start_test_suite(suite_name)
+              test_suite = test_visibility_component.start_test_suite(suite_name)
 
               success = super
               return success unless test_suite
@@ -44,6 +44,10 @@ def run(reporter = ::RSpec::Core::NullReporter)
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb

@@ -18,15 +18,15 @@ def knapsack__run_specs(*args)
                 return super if ::RSpec.configuration.dry_run?
                 return super unless datadog_configuration[:enabled]
 
-                test_session = CI.start_test_session(
+                test_session = test_visibility_component.start_test_session(
                   tags: {
                     CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                     CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                   },
                   service: datadog_configuration[:service_name]
                 )
 
-                test_module = CI.start_test_module(Ext::FRAMEWORK)
+                test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
 
                 result = super
                 return result unless test_module && test_session
@@ -49,6 +49,10 @@ def knapsack__run_specs(*args)
               def datadog_configuration
                 Datadog.configuration.ci[:rspec]
               end
+
+              def test_visibility_component
+                Datadog.send(:components).test_visibility
+              end
             end
           end
         end

lib/datadog/ci/contrib/rspec/runner.rb

@@ -18,15 +18,15 @@ def run_specs(*args)
               return super if ::RSpec.configuration.dry_run?
               return super unless datadog_configuration[:enabled]
 
-              test_session = CI.start_test_session(
+              test_session = test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )
 
-              test_module = CI.start_test_module(Ext::FRAMEWORK)
+              test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
 
               result = super
               return result unless test_module && test_session
@@ -49,6 +49,10 @@ def run_specs(*args)
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -50,6 +50,8 @@ module Datadog
           def ok?: (Cucumber::Core::Test::Result result, untyped strict) -> bool
 
           def configuration: () -> untyped
+
+          def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
         end
       end
     end

sig/datadog/ci/contrib/minitest/runnable.rbs

@@ -16,6 +16,8 @@ module Datadog
             def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
 
             def runnable_methods: () -> Array[String]
+
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
           end
         end
       end

sig/datadog/ci/contrib/minitest/runner.rbs

@@ -13,6 +13,8 @@ module Datadog
             private
 
             def datadog_configuration: () -> untyped
+
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
           end
         end
       end

sig/datadog/ci/contrib/minitest/test.rbs

@@ -23,6 +23,8 @@ module Datadog
 
             def datadog_configuration: () -> untyped
 
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
+
             def finish_with_result: (Datadog::CI::Span? span, String result_code) -> void
           end
         end

sig/datadog/ci/contrib/rspec/example.rbs

@@ -13,6 +13,7 @@ module Datadog
 
             def fetch_top_level_example_group: () -> Hash[Symbol, untyped]
             def datadog_configuration: () -> untyped
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
             def ci_queue?: () -> bool
           end
         end

sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -13,6 +13,8 @@ module Datadog
             private
 
             def datadog_configuration: () -> untyped
+
+            def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
           end
         end
       end

sig/datadog/ci/contrib/rspec/knapsack_pro/runner.rbs

@@ -14,6 +14,8 @@ module Datadog
               private
 
               def datadog_configuration: () -> untyped
+
+              def test_visibility_component: () -> Datadog::CI::TestVisibility::Component
             end
           end
         end