From bba2c38c9066f1a984bac6d95f432c68af270010 Mon Sep 17 00:00:00 2001
From: Andrey Marchenko <andrey.marchenko@datadoghq.com>
Date: Tue, 23 Jan 2024 16:20:22 +0100
Subject: [PATCH] cucumber: better support for pending and undefined scenario:
 report them as skipped, not passed

---
 lib/datadog/ci/contrib/cucumber/formatter.rb  | 12 ++++---
 .../contrib/cucumber/features/passing.feature | 16 ++++++++-
 .../features/step_definitions/steps.rb        |  8 +++++
 .../contrib/cucumber/instrumentation_spec.rb  | 35 +++++++++++++++++++
 vendor/rbs/cucumber/0/cucumber.rbs            |  3 ++
 5 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb
index d5cd2dc9..1f759ec9 100644
--- a/lib/datadog/ci/contrib/cucumber/formatter.rb
+++ b/lib/datadog/ci/contrib/cucumber/formatter.rb
@@ -122,12 +122,16 @@ def test_suite_name(test_case)
           end
 
           def finish_test(span, result)
-            if result.skipped?
-              span.skipped!
-            elsif result.ok?
-              span.passed!
+            if result.skipped? || result.pending? || result.undefined?
+              span.skipped!(reason: result.message)
             elsif result.failed?
               span.failed!(exception: result.exception)
+            elsif result.ok?
+              span.passed!
+            else
+              Datadog.logger.warn do
+                "Unknown test result #{result.class.name} for test #{span.name}"
+              end
             end
             span.finish
           end
diff --git a/spec/datadog/ci/contrib/cucumber/features/passing.feature b/spec/datadog/ci/contrib/cucumber/features/passing.feature
index 38be690a..67814aa1 100644
--- a/spec/datadog/ci/contrib/cucumber/features/passing.feature
+++ b/spec/datadog/ci/contrib/cucumber/features/passing.feature
@@ -3,4 +3,18 @@ Feature: Datadog integration
   Scenario: cucumber scenario
     Given datadog
     And datadog
-    Then datadog
\ No newline at end of file
+    Then datadog
+
+  Scenario: undefined scenario
+    Given datadog
+    And undefined
+    Then undefined
+
+  Scenario: pending scenario
+    Given datadog
+    Then pending
+
+  Scenario: skipped scenario
+    Given datadog
+    And skip
+    Then datadog
diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb
index 9aec38f9..095fb0ca 100644
--- a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb
+++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb
@@ -6,6 +6,14 @@
   expect(1 + 1).to eq(3)
 end
 
+Then "pending" do
+  pending("implementation")
+end
+
+Then "skip" do
+  skip_this_scenario
+end
+
 Then(/I add (-?\d+) and (-?\d+)/) do |n1, n2|
   @res = n1.to_i + n2.to_i
 end
diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb
index a0050612..7e846efe 100644
--- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb
+++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb
@@ -70,6 +70,8 @@
     let(:feature_file_to_run) { "passing.feature" }
 
     it "creates spans for each scenario and step" do
+      expect(test_spans).to have(4).items
+
       scenario_span = spans.find { |s| s.resource == "cucumber scenario" }
 
       expect(scenario_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
@@ -108,6 +110,39 @@
       end
     end
 
+    it "marks undefined cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "undefined scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        'Undefined step: "undefined"'
+      )
+    end
+
+    it "marks pending cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "pending scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        "implementation"
+      )
+    end
+
+    it "marks skipped cucumber scenario as skipped" do
+      undefined_scenario_span = spans.find { |s| s.resource == "skipped scenario" }
+      expect(undefined_scenario_span).not_to be_nil
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+        Datadog::CI::Ext::Test::Status::SKIP
+      )
+      expect(undefined_scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq(
+        "Scenario skipped"
+      )
+    end
+
     it "creates test session span" do
       expect(test_session_span).not_to be_nil
       expect(test_session_span.service).to eq("jalapenos")
diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs
index 6e643260..e6cf0f73 100644
--- a/vendor/rbs/cucumber/0/cucumber.rbs
+++ b/vendor/rbs/cucumber/0/cucumber.rbs
@@ -18,6 +18,9 @@ class Cucumber::Core::Test::Result
   def failed?: () -> bool
   def ok?: () -> bool
   def skipped?: () -> bool
+  def pending?: () -> bool
+  def undefined?: () -> bool
+  def message: () -> String
   def exception: () -> untyped
 end