Merge pull request #90 from DataDog/anmarchenko/cucumber_tslv_instrum…

…entation

[CIVIS-7952] TSLV instrumentation for Cucumber

lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -21,7 +21,7 @@ class Settings < Datadog::CI::Contrib::Settings
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0

lib/datadog/ci/contrib/cucumber/ext.rb

@@ -7,14 +7,16 @@ module Cucumber
         # Cucumber integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "cucumber"
           ENV_ENABLED = "DD_TRACE_CUCUMBER_ENABLED"
-          ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME"
+          DEFAULT_SERVICE_NAME = "cucumber"
+
           FRAMEWORK = "cucumber"
-          OPERATION_NAME = "cucumber.test"
-          SERVICE_NAME = "cucumber"
+
           STEP_SPAN_TYPE = "step"
-          TEST_TYPE = "test"
+
+          # TODO: remove in 1.0
+          ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME"
+          OPERATION_NAME = "cucumber.test"
         end
       end
     end

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -9,31 +9,60 @@ module Contrib
       module Cucumber
         # Defines collection of instrumented Cucumber events
         class Formatter
-          attr_reader :config, :current_feature_span, :current_step_span
+          attr_reader :config
           private :config
-          private :current_feature_span, :current_step_span
 
           def initialize(config)
             @config = config
+            @failed_tests_count = 0
+
+            @current_test_suite = nil
+            @failed_tests_in_current_test_suite = 0
 
             bind_events(config)
           end
 
           def bind_events(config)
+            config.on_event :test_run_started, &method(:on_test_run_started)
+            config.on_event :test_run_finished, &method(:on_test_run_finished)
             config.on_event :test_case_started, &method(:on_test_case_started)
             config.on_event :test_case_finished, &method(:on_test_case_finished)
             config.on_event :test_step_started, &method(:on_test_step_started)
             config.on_event :test_step_finished, &method(:on_test_step_finished)
           end
 
+          def on_test_run_started(event)
+            test_session = CI.start_test_session(
+              tags: {
+                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+              },
+              service: configuration[:service_name]
+            )
+            CI.start_test_module(test_session.name)
+          end
+
+          def on_test_run_finished(event)
+            if event.respond_to?(:success)
+              finish_session(event.success)
+            else
+              finish_session(@failed_tests_count.zero?)
+            end
+          end
+
           def on_test_case_started(event)
+            test_suite_name = event.test_case.location.file
+
+            start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
+
             CI.start_test(
               event.test_case.name,
-              event.test_case.location.file,
+              test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
               },
               service: configuration[:service_name]
             )
@@ -43,15 +72,17 @@ def on_test_case_finished(event)
             test_span = CI.active_test
             return if test_span.nil?
 
-            if event.result.skipped?
-              test_span.skipped!
-            elsif event.result.ok?
-              test_span.passed!
-            elsif event.result.failed?
-              test_span.failed!
+            # We need to track overall test failures manually if we are using cucumber < 8.0 because
+            # TestRunFinished event does not have a success attribute before 8.0.
+            #
+            # To track whether test suite failed or passed we need to
+            # track the number of failed tests in the current test suite.
+            if event.result.failed?
+              @failed_tests_count += 1
+              @failed_tests_in_current_test_suite += 1
             end
 
-            test_span.finish
+            finish_test(test_span, event.result)
           end
 
           def on_test_step_started(event)
@@ -62,18 +93,67 @@ def on_test_step_finished(event)
             current_step_span = CI.active_span(Ext::STEP_SPAN_TYPE)
             return if current_step_span.nil?
 
-            if event.result.skipped?
-              current_step_span.skipped!
-            elsif event.result.ok?
-              current_step_span.passed!
-            elsif event.result.failed?
-              current_step_span.failed!(exception: event.result.exception)
+            finish_test(current_step_span, event.result)
+          end
+
+          private
+
+          def finish_test(span, result)
+            if result.skipped?
+              span.skipped!
+            elsif result.ok?
+              span.passed!
+            elsif result.failed?
+              span.failed!(exception: result.exception)
             end
+            span.finish
+          end
+
+          def finish_session(result)
+            finish_current_test_suite
+
+            test_session = CI.active_test_session
+            test_module = CI.active_test_module
+
+            return unless test_session && test_module
 
-            current_step_span.finish
+            if result
+              test_module.passed!
+              test_session.passed!
+            else
+              test_module.failed!
+              test_session.failed!
+            end
+
+            test_module.finish
+            test_session.finish
           end
 
-          private
+          def start_test_suite(test_suite_name)
+            finish_current_test_suite
+
+            @current_test_suite = CI.start_test_suite(test_suite_name)
+          end
+
+          def finish_current_test_suite
+            test_suite = @current_test_suite
+            return unless test_suite
+
+            if @failed_tests_in_current_test_suite.zero?
+              test_suite.passed!
+            else
+              test_suite.failed!
+            end
+            @failed_tests_in_current_test_suite = 0
+            test_suite.finish
+          end
+
+          def same_test_suite_as_current?(test_suite_name)
+            test_suite = @current_test_suite
+            return false unless test_suite
+
+            test_suite.name == test_suite_name
+          end
 
           def configuration
             Datadog.configuration.ci[:cucumber]

sig/datadog/ci/contrib/cucumber/ext.rbs

@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Cucumber
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,11 +11,9 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
+          DEFAULT_SERVICE_NAME: String
 
           STEP_SPAN_TYPE: String
-
-          TEST_TYPE: String
         end
       end
     end

sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -4,19 +4,22 @@ module Datadog
       module Cucumber
         class Formatter
           private
+          @failed_tests_count: Integer
+          @current_test_suite: Datadog::CI::Span?
+          @failed_tests_in_current_test_suite: Integer
 
           attr_reader config: untyped
 
-          attr_reader current_feature_span: untyped
-
-          attr_reader current_step_span: untyped
-
           public
 
           def initialize: (untyped config) -> void
 
           def bind_events: (untyped config) -> untyped
 
+          def on_test_run_started: (untyped event) -> untyped
+
+          def on_test_run_finished: (untyped event) -> untyped
+
           def on_test_case_started: (untyped event) -> untyped
 
           def on_test_case_finished: (untyped event) -> (nil | untyped)
@@ -27,6 +30,16 @@ module Datadog
 
           private
 
+          def start_test_suite: (String test_suite_name) -> void
+
+          def finish_current_test_suite: () -> void
+
+          def same_test_suite_as_current?: (String test_suite_name) -> bool
+
+          def finish_session: (bool result) -> void
+
+          def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void
+
           def configuration: () -> untyped
         end
       end

spec/datadog/ci/contrib/cucumber/features/failing.feature

@@ -0,0 +1,5 @@
+Feature: Datadog integration
+  Scenario: cucumber failing scenario
+    Given datadog
+    And datadog
+    Then failure

spec/datadog/ci/contrib/cucumber/cucumber.features → spec/datadog/ci/contrib/cucumber/features/passing.feature

@@ -2,3 +2,5 @@ Feature: Datadog integration
 
   Scenario: cucumber scenario
     Given datadog
+    And datadog
+    Then datadog

spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb

@@ -0,0 +1,7 @@
+Then "datadog" do
+  true
+end
+
+Then "failure" do
+  expect(1 + 1).to eq(3)
+end