cucumber: better support for pending and undefined scenario: report t…

…hem as skipped, not passed

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -122,12 +122,16 @@ def test_suite_name(test_case)
           end
 
           def finish_test(span, result)
-            if result.skipped?
-              span.skipped!
-            elsif result.ok?
-              span.passed!
+            if result.skipped? || result.pending? || result.undefined?
+              span.skipped!(reason: result.message)
             elsif result.failed?
               span.failed!(exception: result.exception)
+            elsif result.ok?
+              span.passed!
+            else
+              Datadog.logger.warn do
+                "Unknown test result #{result.class.name} for test #{span.name}"
+              end
             end
             span.finish
           end

spec/datadog/ci/contrib/cucumber/features/passing.feature

@@ -3,4 +3,18 @@ Feature: Datadog integration
   Scenario: cucumber scenario
     Given datadog
     And datadog
-    Then datadog
+    Then datadog
+
+  Scenario: undefined scenario
+    Given datadog
+    And undefined
+    Then undefined
+
+  Scenario: pending scenario
+    Given datadog
+    Then pending
+
+  Scenario: skipped scenario
+    Given datadog
+    And skip
+    Then datadog

spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb

@@ -6,6 +6,14 @@
   expect(1 + 1).to eq(3)
 end
 
+Then "pending" do
+  pending("implementation")
+end
+
+Then "skip" do
+  skip_this_scenario
+end
+
 Then(/I add (-?\d+) and (-?\d+)/) do |n1, n2|
   @res = n1.to_i + n2.to_i
 end

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -70,6 +70,8 @@
     let(:feature_file_to_run) { "passing.feature" }
 
     it "creates spans for each scenario and step" do
+      expect(test_spans).to have(4).items
+
       scenario_span = spans.find { |s| s.resource == "cucumber scenario" }
 
       expect(scenario_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
@@ -108,6 +110,39 @@
       end
     end
 
+    it "marks undefined cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "undefined scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        'Undefined step: "undefined"'
+      )
+    end
+
+    it "marks pending cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "pending scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        "implementation"
+      )
+    end
+
+    it "marks skipped cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "skipped scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        "Scenario skipped"
+      )
+    end
+
     it "creates test session span" do
       expect(test_session_span).not_to be_nil
       expect(test_session_span.service).to eq("jalapenos")

vendor/rbs/cucumber/0/cucumber.rbs

@@ -18,6 +18,9 @@ class Cucumber::Core::Test::Result
   def failed?: () -> bool
   def ok?: () -> bool
   def skipped?: () -> bool
+  def pending?: () -> bool
+  def undefined?: () -> bool
+  def message: () -> String
   def exception: () -> untyped
 end