From 46ce9176fdc83c75dd523e806696430eab628f38 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 10 Jan 2024 15:24:24 +0100 Subject: [PATCH 1/4] add cucumber feature with examples to instrumentation specs --- .../features/step_definitions/steps.rb | 8 ++++ .../cucumber/features/with_parameters.feature | 12 ++++++ .../contrib/cucumber/instrumentation_spec.rb | 40 ++++++++++++++++--- 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 spec/datadog/ci/contrib/cucumber/features/with_parameters.feature diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb index b56a3d96..9aec38f9 100644 --- a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb +++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb @@ -5,3 +5,11 @@ Then "failure" do expect(1 + 1).to eq(3) end + +Then(/I add (-?\d+) and (-?\d+)/) do |n1, n2| + @res = n1.to_i + n2.to_i +end + +Then(/the result should be (-?\d+)/) do |res| + expect(@res).to eq(res.to_i) +end diff --git a/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature b/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature new file mode 100644 index 00000000..a32f1c73 --- /dev/null +++ b/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature @@ -0,0 +1,12 @@ +Feature: Datadog integration for parametrized tests + + Scenario Outline: scenario with examples + Given datadog + When I add and + Then the result should be + + Examples: + | num1 | num2 | total | + | -1 | 1 | 0 | + | 1 | 1 | 2 | + | 0 | 0 | 0 | \ No newline at end of file diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 9f0f26aa..f18aba2c 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -10,6 +10,9 @@ let(:integration_options) { {service_name: "jalapenos"} } end + let(:cucumber_8_or_above) { Gem::Version.new("8.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version } + let(:cucumber_4_or_above) { Gem::Version.new("4.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version } + let(:run_id) { rand(1..2**64 - 1) } let(:steps_file_definition_path) { "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb" } let(:steps_file_for_run_path) do @@ -35,12 +38,10 @@ let(:kernel) { double(:kernel) } let(:cli) do - cucumber_8 = Gem::Version.new("8.0.0") - - if Datadog::CI::Contrib::Cucumber::Integration.version < cucumber_8 - Cucumber::Cli::Main.new(args, stdin, stdout, stderr, kernel) - else + if cucumber_8_or_above Cucumber::Cli::Main.new(args, stdout, stderr, kernel) + else + Cucumber::Cli::Main.new(args, stdin, stdout, stderr, kernel) end end @@ -202,6 +203,33 @@ end end + context "executing a scenario with examples" do + let(:feature_file_to_run) { "with_parameters.feature" } + + it "a single test suite with a test span for each example" do + expect(test_spans.count).to eq(3) + expect(test_suite_spans.count).to eq(1) + + test_spans.each_with_index do |span, index| + # naming for scenarios changed since cucumber 4 + if cucumber_4_or_above + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("scenario with examples") + else + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq( + "scenario with examples, Examples (##{index + 1})" + ) + end + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( + "spec/datadog/ci/contrib/cucumber/features/with_parameters.feature" + ) + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s) + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + end + end + end + context "executing several features at once" do let(:expected_test_run_code) { 2 } @@ -209,7 +237,7 @@ let(:failing_test_suite) { test_suite_spans.find { |span| span.name =~ /failing/ } } it "creates a test suite span for each feature" do - expect(test_suite_spans.count).to eq(2) + expect(test_suite_spans.count).to eq(3) expect(passing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::PASS ) From 95fdc8b8c3a804879d0ddc9ff64ced5c8e71a225 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 10 Jan 2024 15:34:07 +0100 Subject: [PATCH 2/4] add test.parameters tag to constants --- lib/datadog/ci/ext/test.rb | 1 + sig/datadog/ci/ext/test.rbs | 2 ++ .../ci/contrib/cucumber/features/with_parameters.feature | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index 1aa0a46c..d5cf582d 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -22,6 +22,7 @@ module Test TAG_SOURCE_FILE = "test.source.file" TAG_SOURCE_START = "test.source.start" TAG_CODEOWNERS = "test.codeowners" + TAG_PARAMETERS = "test.parameters" TEST_TYPE = "test" diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 95822b88..501e596f 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -32,6 +32,8 @@ module Datadog TAG_CODEOWNERS: String + TAG_PARAMETERS: String + TAG_TEST_SESSION_ID: String TAG_TEST_MODULE_ID: String diff --git a/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature b/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature index a32f1c73..03f97e1f 100644 --- a/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature +++ b/spec/datadog/ci/contrib/cucumber/features/with_parameters.feature @@ -7,6 +7,6 @@ Feature: Datadog integration for parametrized tests Examples: | num1 | num2 | total | - | -1 | 1 | 0 | - | 1 | 1 | 2 | - | 0 | 0 | 0 | \ No newline at end of file + | 0 | 1 | 1 | + | 1 | 2 | 3 | + | 2 | 3 | 5 | \ No newline at end of file From 3e0d74efdc4225ad6b539f94aa319667edc3e080 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 11 Jan 2024 12:03:01 +0100 Subject: [PATCH 3/4] add test.parameters support for cucumber --- lib/datadog/ci/contrib/cucumber/formatter.rb | 45 ++++++++++++++++--- lib/datadog/ci/ext/test.rb | 3 +- sig/datadog/ci/contrib/cucumber/formatter.rbs | 3 ++ sig/datadog/ci/ext/test.rbs | 2 - .../contrib/cucumber/instrumentation_spec.rb | 8 +++- vendor/rbs/cucumber/0/cucumber.rbs | 10 +++++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 9a745406..0efe4153 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "json" + require_relative "../../ext/test" require_relative "../../utils/git" require_relative "ext" @@ -14,6 +16,8 @@ class Formatter private :config def initialize(config) + @ast_lookup = ::Cucumber::Formatter::AstLookup.new(config) if defined?(::Cucumber::Formatter::AstLookup) + @config = config @failed_tests_count = 0 @@ -54,18 +58,24 @@ def on_test_run_finished(event) def on_test_case_started(event) test_suite_name = event.test_case.location.file + tags = { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE, + CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file), + CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s + } + + if (parameters = extract_parameters_hash(event.test_case)) + tags[CI::Ext::Test::TAG_PARAMETERS] = JSON.generate(parameters) + end + start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name) CI.start_test( event.test_case.name, test_suite_name, - tags: { - CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, - CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE, - CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file), - CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s - }, + tags: tags, service: configuration[:service_name] ) end @@ -155,6 +165,27 @@ def same_test_suite_as_current?(test_suite_name) test_suite.name == test_suite_name end + def extract_parameters_hash(test_case) + # not supported in cucumber < 4.0 + return nil unless @ast_lookup + + scenario_source = @ast_lookup.scenario_source(test_case) + + # cucumber examples are only supported for scenario outlines + return nil unless scenario_source.type == :ScenarioOutline + + scenario_source.examples.table_header.cells.map(&:value).zip( + scenario_source.row.cells.map(&:value) + ).to_h + rescue => e + Datadog.logger.warn do + "Unable to extract parameters from test case #{test_case.name}: " \ + "#{e.class.name} #{e.message} at #{Array(e.backtrace).first}" + end + + nil + end + def configuration Datadog.configuration.ci[:cucumber] end diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index d5cf582d..f242d791 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -12,11 +12,10 @@ module Test TAG_FRAMEWORK = "test.framework" TAG_FRAMEWORK_VERSION = "test.framework_version" TAG_NAME = "test.name" - TAG_SKIP_REASON = "test.skip_reason" # DEV: Not populated yet + TAG_SKIP_REASON = "test.skip_reason" TAG_STATUS = "test.status" TAG_SUITE = "test.suite" TAG_MODULE = "test.module" - TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" TAG_COMMAND = "test.command" TAG_SOURCE_FILE = "test.source.file" diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 5cc495c9..216e7569 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -6,6 +6,7 @@ module Datadog private @failed_tests_count: Integer @current_test_suite: Datadog::CI::Span? + @ast_lookup: ::Cucumber::Formatter::AstLookup attr_reader config: untyped @@ -39,6 +40,8 @@ module Datadog def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void + def extract_parameters_hash: (untyped test_case) -> Hash[String, String]? + def configuration: () -> untyped end end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 501e596f..e9d9ba42 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -20,8 +20,6 @@ module Datadog TAG_MODULE: String - TAG_TRAITS: String - TAG_TYPE: String TAG_COMMAND: String diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index f18aba2c..c72b32bb 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -206,14 +206,18 @@ context "executing a scenario with examples" do let(:feature_file_to_run) { "with_parameters.feature" } - it "a single test suite with a test span for each example" do + it "a single test suite, and a test span for each example with parameters JSON" do expect(test_spans.count).to eq(3) expect(test_suite_spans.count).to eq(1) test_spans.each_with_index do |span, index| - # naming for scenarios changed since cucumber 4 + # test parameters are available since cucumber 4 if cucumber_4_or_above expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("scenario with examples") + + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_PARAMETERS)).to eq( + "{\"num1\":\"#{index}\",\"num2\":\"#{index + 1}\",\"total\":\"#{index + index + 1}\"}" + ) else expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq( "scenario with examples, Examples (##{index + 1})" diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index 6dbead93..45f541ac 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -11,9 +11,19 @@ class Cucumber::Runtime def formatters: () -> untyped end +module Cucumber::Formatter +end + + class Cucumber::Core::Test::Result def failed?: () -> bool def ok?: () -> bool def skipped?: () -> bool def exception: () -> untyped +end + +class Cucumber::Formatter::AstLookup + def initialize: (untyped config) -> void + + def scenario_source: (untyped test_case) -> untyped end \ No newline at end of file From fd793add047fd9c705ec22506afa495ae0b65370 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 11 Jan 2024 14:17:16 +0100 Subject: [PATCH 4/4] set the parameters for spans in JSON object with arguments and metadata fields --- lib/datadog/ci/contrib/cucumber/formatter.rb | 12 +++++------- .../ci/contrib/cucumber/instrumentation.rb | 3 ++- lib/datadog/ci/null_span.rb | 3 +++ lib/datadog/ci/span.rb | 19 +++++++++++++++++++ lib/datadog/ci/test.rb | 2 ++ sig/datadog/ci/span.rbs | 2 ++ .../contrib/cucumber/instrumentation_spec.rb | 2 +- spec/datadog/ci/span_spec.rb | 12 ++++++++++++ 8 files changed, 46 insertions(+), 9 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 0efe4153..81a21273 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "json" - require_relative "../../ext/test" require_relative "../../utils/git" require_relative "ext" @@ -66,18 +64,18 @@ def on_test_case_started(event) CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s } - if (parameters = extract_parameters_hash(event.test_case)) - tags[CI::Ext::Test::TAG_PARAMETERS] = JSON.generate(parameters) - end - start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name) - CI.start_test( + test_span = CI.start_test( event.test_case.name, test_suite_name, tags: tags, service: configuration[:service_name] ) + + if (parameters = extract_parameters_hash(event.test_case)) + test_span.set_parameters(parameters) + end end def on_test_case_finished(event) diff --git a/lib/datadog/ci/contrib/cucumber/instrumentation.rb b/lib/datadog/ci/contrib/cucumber/instrumentation.rb index 9e98cc58..7b4a402d 100644 --- a/lib/datadog/ci/contrib/cucumber/instrumentation.rb +++ b/lib/datadog/ci/contrib/cucumber/instrumentation.rb @@ -17,8 +17,9 @@ module InstanceMethods attr_reader :datadog_formatter def formatters + existing_formatters = super @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration) - [@datadog_formatter] + super + [@datadog_formatter] + existing_formatters end end end diff --git a/lib/datadog/ci/null_span.rb b/lib/datadog/ci/null_span.rb index e010af77..348d4ccf 100644 --- a/lib/datadog/ci/null_span.rb +++ b/lib/datadog/ci/null_span.rb @@ -55,6 +55,9 @@ def set_environment_runtime_tags def set_default_tags end + def set_parameters(arguments, metadata = {}) + end + def to_s self.class.to_s end diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index b038611e..6bf9dd3f 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -133,6 +133,25 @@ def set_default_tags tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST) end + # Sets the parameters for this span for parametrized tests (e.g. Cucumber examples or RSpec shared specs). + # Parameters are needed to compute test fingerprint to distinguish between different tests having same names. + # @param [Hash] arguments the arguments that test accepts as key-value hash + # @param [Hash] metadata optional metadata + # @return [void] + def set_parameters(arguments, metadata = {}) + return if arguments.nil? || arguments.empty? + + set_tag( + Ext::Test::TAG_PARAMETERS, + JSON.generate( + { + arguments: arguments, + metadata: metadata + } + ) + ) + end + def to_s "#{self.class}(name:#{name},tracer_span:#{@tracer_span})" end diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index de34c4db..8b9a3f99 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "json" + require_relative "span" module Datadog diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs index b3a7a5a0..da6e8346 100644 --- a/sig/datadog/ci/span.rbs +++ b/sig/datadog/ci/span.rbs @@ -42,6 +42,8 @@ module Datadog def set_environment_runtime_tags: () -> void def set_default_tags: () -> void + + def set_parameters: (Hash[String, Object] arguments, ?Hash[String, Object] metadata) -> void end end end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index c72b32bb..1161ecb1 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -216,7 +216,7 @@ expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("scenario with examples") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_PARAMETERS)).to eq( - "{\"num1\":\"#{index}\",\"num2\":\"#{index + 1}\",\"total\":\"#{index + index + 1}\"}" + "{\"arguments\":{\"num1\":\"#{index}\",\"num2\":\"#{index + 1}\",\"total\":\"#{index + index + 1}\"},\"metadata\":{}}" ) else expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq( diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index 5b12e575..d8745fa5 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -228,4 +228,16 @@ expect(span.span_type).to eq("test") end end + + describe "#set_parameters" do + let(:parameters) { {"foo" => "bar", "baz" => "qux"} } + + it "sets the parameters" do + expect(tracer_span).to receive(:set_tag).with( + "test.parameters", JSON.generate({arguments: parameters, metadata: {}}) + ) + + span.set_parameters(parameters) + end + end end