From 3e0d74efdc4225ad6b539f94aa319667edc3e080 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 11 Jan 2024 12:03:01 +0100 Subject: [PATCH] add test.parameters support for cucumber --- lib/datadog/ci/contrib/cucumber/formatter.rb | 45 ++++++++++++++++--- lib/datadog/ci/ext/test.rb | 3 +- sig/datadog/ci/contrib/cucumber/formatter.rbs | 3 ++ sig/datadog/ci/ext/test.rbs | 2 - .../contrib/cucumber/instrumentation_spec.rb | 8 +++- vendor/rbs/cucumber/0/cucumber.rbs | 10 +++++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 9a745406..0efe4153 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "json" + require_relative "../../ext/test" require_relative "../../utils/git" require_relative "ext" @@ -14,6 +16,8 @@ class Formatter private :config def initialize(config) + @ast_lookup = ::Cucumber::Formatter::AstLookup.new(config) if defined?(::Cucumber::Formatter::AstLookup) + @config = config @failed_tests_count = 0 @@ -54,18 +58,24 @@ def on_test_run_finished(event) def on_test_case_started(event) test_suite_name = event.test_case.location.file + tags = { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE, + CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file), + CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s + } + + if (parameters = extract_parameters_hash(event.test_case)) + tags[CI::Ext::Test::TAG_PARAMETERS] = JSON.generate(parameters) + end + start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name) CI.start_test( event.test_case.name, test_suite_name, - tags: { - CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, - CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE, - CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file), - CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s - }, + tags: tags, service: configuration[:service_name] ) end @@ -155,6 +165,27 @@ def same_test_suite_as_current?(test_suite_name) test_suite.name == test_suite_name end + def extract_parameters_hash(test_case) + # not supported in cucumber < 4.0 + return nil unless @ast_lookup + + scenario_source = @ast_lookup.scenario_source(test_case) + + # cucumber examples are only supported for scenario outlines + return nil unless scenario_source.type == :ScenarioOutline + + scenario_source.examples.table_header.cells.map(&:value).zip( + scenario_source.row.cells.map(&:value) + ).to_h + rescue => e + Datadog.logger.warn do + "Unable to extract parameters from test case #{test_case.name}: " \ + "#{e.class.name} #{e.message} at #{Array(e.backtrace).first}" + end + + nil + end + def configuration Datadog.configuration.ci[:cucumber] end diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index d5cf582d..f242d791 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -12,11 +12,10 @@ module Test TAG_FRAMEWORK = "test.framework" TAG_FRAMEWORK_VERSION = "test.framework_version" TAG_NAME = "test.name" - TAG_SKIP_REASON = "test.skip_reason" # DEV: Not populated yet + TAG_SKIP_REASON = "test.skip_reason" TAG_STATUS = "test.status" TAG_SUITE = "test.suite" TAG_MODULE = "test.module" - TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" TAG_COMMAND = "test.command" TAG_SOURCE_FILE = "test.source.file" diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 5cc495c9..216e7569 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -6,6 +6,7 @@ module Datadog private @failed_tests_count: Integer @current_test_suite: Datadog::CI::Span? + @ast_lookup: ::Cucumber::Formatter::AstLookup attr_reader config: untyped @@ -39,6 +40,8 @@ module Datadog def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void + def extract_parameters_hash: (untyped test_case) -> Hash[String, String]? + def configuration: () -> untyped end end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 501e596f..e9d9ba42 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -20,8 +20,6 @@ module Datadog TAG_MODULE: String - TAG_TRAITS: String - TAG_TYPE: String TAG_COMMAND: String diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index f18aba2c..c72b32bb 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -206,14 +206,18 @@ context "executing a scenario with examples" do let(:feature_file_to_run) { "with_parameters.feature" } - it "a single test suite with a test span for each example" do + it "a single test suite, and a test span for each example with parameters JSON" do expect(test_spans.count).to eq(3) expect(test_suite_spans.count).to eq(1) test_spans.each_with_index do |span, index| - # naming for scenarios changed since cucumber 4 + # test parameters are available since cucumber 4 if cucumber_4_or_above expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("scenario with examples") + + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_PARAMETERS)).to eq( + "{\"num1\":\"#{index}\",\"num2\":\"#{index + 1}\",\"total\":\"#{index + index + 1}\"}" + ) else expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq( "scenario with examples, Examples (##{index + 1})" diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index 6dbead93..45f541ac 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -11,9 +11,19 @@ class Cucumber::Runtime def formatters: () -> untyped end +module Cucumber::Formatter +end + + class Cucumber::Core::Test::Result def failed?: () -> bool def ok?: () -> bool def skipped?: () -> bool def exception: () -> untyped +end + +class Cucumber::Formatter::AstLookup + def initialize: (untyped config) -> void + + def scenario_source: (untyped test_case) -> untyped end \ No newline at end of file