From 1559c3e26718b9e4fad55eb63968517e17d744ee Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 13 Dec 2023 13:57:51 +0100 Subject: [PATCH 01/11] session and module instrumentation for Cucumber --- lib/datadog/ci/contrib/cucumber/ext.rb | 6 ++- lib/datadog/ci/contrib/cucumber/formatter.rb | 41 ++++++++++++++- sig/datadog/ci/contrib/cucumber/formatter.rbs | 8 +-- .../contrib/cucumber/instrumentation_spec.rb | 51 +++++++++++++++++++ 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/ext.rb b/lib/datadog/ci/contrib/cucumber/ext.rb index 4878a445..6182b408 100644 --- a/lib/datadog/ci/contrib/cucumber/ext.rb +++ b/lib/datadog/ci/contrib/cucumber/ext.rb @@ -9,12 +9,14 @@ module Cucumber module Ext APP = "cucumber" ENV_ENABLED = "DD_TRACE_CUCUMBER_ENABLED" - ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME" FRAMEWORK = "cucumber" OPERATION_NAME = "cucumber.test" SERVICE_NAME = "cucumber" - STEP_SPAN_TYPE = "step" TEST_TYPE = "test" + STEP_SPAN_TYPE = "step" + + # TODO: remove in 1.0 + ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME" end end end diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 2a2f3e34..1d2522f4 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -9,9 +9,8 @@ module Contrib module Cucumber # Defines collection of instrumented Cucumber events class Formatter - attr_reader :config, :current_feature_span, :current_step_span + attr_reader :config private :config - private :current_feature_span, :current_step_span def initialize(config) @config = config @@ -20,12 +19,50 @@ def initialize(config) end def bind_events(config) + config.on_event :test_run_started, &method(:on_test_run_started) + config.on_event :test_run_finished, &method(:on_test_run_finished) config.on_event :test_case_started, &method(:on_test_case_started) config.on_event :test_case_finished, &method(:on_test_case_finished) config.on_event :test_step_started, &method(:on_test_step_started) config.on_event :test_step_finished, &method(:on_test_step_finished) end + def on_test_run_started(event) + p "TEST RUN!" + test_session = CI.start_test_session( + tags: { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, + CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE + }, + service: configuration[:service_name] + ) + CI.start_test_module(test_session.name) + end + + def on_test_run_finished(event) + test_session = CI.active_test_session + test_module = CI.active_test_module + + if test_session && test_module + if event.respond_to?(:success) + if event.success + test_module.passed! + test_session.passed! + else + test_module.failed! + test_session.failed! + end + else + # we need to track results manually if we are using cucumber < 8.0 + test_module.passed! + test_session.passed! + end + test_module.finish + test_session.finish + end + end + def on_test_case_started(event) CI.start_test( event.test_case.name, diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 30143e0a..46e2e712 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -7,16 +7,16 @@ module Datadog attr_reader config: untyped - attr_reader current_feature_span: untyped - - attr_reader current_step_span: untyped - public def initialize: (untyped config) -> void def bind_events: (untyped config) -> untyped + def on_test_run_started: (untyped event) -> untyped + + def on_test_run_finished: (untyped event) -> untyped + def on_test_case_started: (untyped event) -> untyped def on_test_case_finished: (untyped event) -> (nil | untyped) diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 8ad3a25a..82b12f89 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -48,7 +48,11 @@ def do_execute expect(scenario_span.resource).to eq("cucumber scenario") expect(scenario_span.service).to eq("jalapenos") expect(scenario_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) expect(scenario_span.name).to eq("cucumber scenario") + expect(step_span.resource).to eq("datadog") spans.each do |span| @@ -56,5 +60,52 @@ def do_execute .to eq(Datadog::CI::Ext::Test::CONTEXT_ORIGIN) end end + + it "creates test sesion span" do + expect(kernel).to receive(:exit).with(0) + + do_execute + + expect(test_session_span).not_to be_nil + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::TEST_TYPE + ) + expect(test_session_span.service).to eq("jalapenos") + end + + it "creates test module span" do + expect(kernel).to receive(:exit).with(0) + + do_execute + + expect(test_module_span).not_to be_nil + expect(test_module_span.name).to eq(test_session_span.name) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::TEST_TYPE + ) + expect(test_module_span.service).to eq("jalapenos") + end + + it "connects scenario span to test session and test module" do + expect(kernel).to receive(:exit).with(0) + + do_execute + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_module_span.name) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) + end end end From ca9ffe050106eb306abd2fe01eddfa375c9d8bf1 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 13 Dec 2023 16:08:26 +0100 Subject: [PATCH 02/11] reformat cucumber tests --- lib/datadog/ci/contrib/cucumber/formatter.rb | 1 - .../{cucumber.features => features/passing.feature} | 0 .../cucumber/features/step_definitions/steps.rb | 6 ++++++ .../ci/contrib/cucumber/instrumentation_spec.rb | 11 +++++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) rename spec/datadog/ci/contrib/cucumber/{cucumber.features => features/passing.feature} (100%) create mode 100644 spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 1d2522f4..a36e793f 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -28,7 +28,6 @@ def bind_events(config) end def on_test_run_started(event) - p "TEST RUN!" test_session = CI.start_test_session( tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, diff --git a/spec/datadog/ci/contrib/cucumber/cucumber.features b/spec/datadog/ci/contrib/cucumber/features/passing.feature similarity index 100% rename from spec/datadog/ci/contrib/cucumber/cucumber.features rename to spec/datadog/ci/contrib/cucumber/features/passing.feature diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb new file mode 100644 index 00000000..15e7b50b --- /dev/null +++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb @@ -0,0 +1,6 @@ +Given "datadog" do +end + +Then "failure" do + expect(1 + 1).to eq(3) +end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 82b12f89..31ac3540 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -28,8 +28,14 @@ end end - context "executing a test suite" do - let(:args) { ["spec/datadog/ci/contrib/cucumber/cucumber.features"] } + context "executing a passing test suite" do + let(:args) do + [ + "-r", + "spec/datadog/ci/contrib/cucumber/features/step_definitions", + "spec/datadog/ci/contrib/cucumber/features/passing.feature" + ] + end def do_execute cli.execute!(existing_runtime) @@ -52,6 +58,7 @@ def do_execute Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK ) expect(scenario_span.name).to eq("cucumber scenario") + # expect(scenario_span) expect(step_span.resource).to eq("datadog") From fb74e019dfc992ceea7fe8df2b672e7b062d62aa Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 13 Dec 2023 16:26:52 +0100 Subject: [PATCH 03/11] reorganize cucumber constants, more assertions for cucumber instrumentation --- .../cucumber/configuration/settings.rb | 2 +- lib/datadog/ci/contrib/cucumber/ext.rb | 8 ++-- lib/datadog/ci/contrib/cucumber/formatter.rb | 4 +- sig/datadog/ci/contrib/cucumber/ext.rbs | 6 +-- .../contrib/cucumber/instrumentation_spec.rb | 40 ++++++++++++++----- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/configuration/settings.rb b/lib/datadog/ci/contrib/cucumber/configuration/settings.rb index 3e916f11..e99a4948 100644 --- a/lib/datadog/ci/contrib/cucumber/configuration/settings.rb +++ b/lib/datadog/ci/contrib/cucumber/configuration/settings.rb @@ -21,7 +21,7 @@ class Settings < Datadog::CI::Contrib::Settings option :service_name do |o| o.type :string - o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME } + o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME } end # @deprecated Will be removed in 1.0 diff --git a/lib/datadog/ci/contrib/cucumber/ext.rb b/lib/datadog/ci/contrib/cucumber/ext.rb index 6182b408..f8043a1f 100644 --- a/lib/datadog/ci/contrib/cucumber/ext.rb +++ b/lib/datadog/ci/contrib/cucumber/ext.rb @@ -7,16 +7,16 @@ module Cucumber # Cucumber integration constants # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes. module Ext - APP = "cucumber" ENV_ENABLED = "DD_TRACE_CUCUMBER_ENABLED" + DEFAULT_SERVICE_NAME = "cucumber" + FRAMEWORK = "cucumber" - OPERATION_NAME = "cucumber.test" - SERVICE_NAME = "cucumber" - TEST_TYPE = "test" + STEP_SPAN_TYPE = "step" # TODO: remove in 1.0 ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME" + OPERATION_NAME = "cucumber.test" end end end diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index a36e793f..7f2bf6f5 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -32,7 +32,7 @@ def on_test_run_started(event) tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE }, service: configuration[:service_name] ) @@ -69,7 +69,7 @@ def on_test_case_started(event) tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE }, service: configuration[:service_name] ) diff --git a/sig/datadog/ci/contrib/cucumber/ext.rbs b/sig/datadog/ci/contrib/cucumber/ext.rbs index 112bcda7..18d81294 100644 --- a/sig/datadog/ci/contrib/cucumber/ext.rbs +++ b/sig/datadog/ci/contrib/cucumber/ext.rbs @@ -3,8 +3,6 @@ module Datadog module Contrib module Cucumber module Ext - APP: String - ENV_ENABLED: String ENV_OPERATION_NAME: String @@ -13,11 +11,9 @@ module Datadog OPERATION_NAME: String - SERVICE_NAME: String + DEFAULT_SERVICE_NAME: String STEP_SPAN_TYPE: String - - TEST_TYPE: String end end end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 31ac3540..10787fbe 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -49,17 +49,27 @@ def do_execute do_execute scenario_span = spans.find { |s| s.resource == "cucumber scenario" } - step_span = spans.find { |s| s.resource == "datadog" } + expect(scenario_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) + expect(scenario_span.name).to eq("cucumber scenario") expect(scenario_span.resource).to eq("cucumber scenario") expect(scenario_span.service).to eq("jalapenos") - expect(scenario_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) + + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("cucumber scenario") + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( + "spec/datadog/ci/contrib/cucumber/features/passing.feature" + ) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE) expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK ) - expect(scenario_span.name).to eq("cucumber scenario") - # expect(scenario_span) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + step_span = spans.find { |s| s.resource == "datadog" } expect(step_span.resource).to eq("datadog") spans.each do |span| @@ -68,12 +78,16 @@ def do_execute end end - it "creates test sesion span" do + it "creates test session span" do expect(kernel).to receive(:exit).with(0) do_execute expect(test_session_span).not_to be_nil + expect(test_session_span.service).to eq("jalapenos") + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK ) @@ -81,9 +95,9 @@ def do_execute Datadog::CI::Contrib::Cucumber::Integration.version.to_s ) expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( - Datadog::CI::Contrib::Cucumber::Ext::TEST_TYPE + Datadog::CI::Ext::Test::TEST_TYPE ) - expect(test_session_span.service).to eq("jalapenos") + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) end it "creates test module span" do @@ -92,7 +106,11 @@ def do_execute do_execute expect(test_module_span).not_to be_nil - expect(test_module_span.name).to eq(test_session_span.name) + expect(test_module_span.name).to eq(test_command) + expect(test_module_span.service).to eq("jalapenos") + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK ) @@ -100,9 +118,9 @@ def do_execute Datadog::CI::Contrib::Cucumber::Integration.version.to_s ) expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( - Datadog::CI::Contrib::Cucumber::Ext::TEST_TYPE + Datadog::CI::Ext::Test::TEST_TYPE ) - expect(test_module_span.service).to eq("jalapenos") + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) end it "connects scenario span to test session and test module" do @@ -111,7 +129,7 @@ def do_execute do_execute expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s) - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_module_span.name) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_command) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) end end From 76488fcac6208caf674a68e4e33624444ff2bf24 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 13 Dec 2023 17:50:09 +0100 Subject: [PATCH 04/11] make Ruby load Cucumber step definitions on every Rspec example run --- .../contrib/cucumber/features/failing.feature | 5 ++ .../contrib/cucumber/features/passing.feature | 2 + .../features/step_definitions/steps.rb | 3 +- .../contrib/cucumber/instrumentation_spec.rb | 59 +++++++++++++++++-- 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 spec/datadog/ci/contrib/cucumber/features/failing.feature diff --git a/spec/datadog/ci/contrib/cucumber/features/failing.feature b/spec/datadog/ci/contrib/cucumber/features/failing.feature new file mode 100644 index 00000000..518b6ed3 --- /dev/null +++ b/spec/datadog/ci/contrib/cucumber/features/failing.feature @@ -0,0 +1,5 @@ +Feature: Datadog integration + Scenario: cucumber failing scenario + Given datadog + And datadog + Then failure diff --git a/spec/datadog/ci/contrib/cucumber/features/passing.feature b/spec/datadog/ci/contrib/cucumber/features/passing.feature index 4b5209c9..38be690a 100644 --- a/spec/datadog/ci/contrib/cucumber/features/passing.feature +++ b/spec/datadog/ci/contrib/cucumber/features/passing.feature @@ -2,3 +2,5 @@ Feature: Datadog integration Scenario: cucumber scenario Given datadog + And datadog + Then datadog \ No newline at end of file diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb index 15e7b50b..b56a3d96 100644 --- a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb +++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb @@ -1,4 +1,5 @@ -Given "datadog" do +Then "datadog" do + true end Then "failure" do diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 10787fbe..1f1244d2 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -1,14 +1,35 @@ require "stringio" +require "fileutils" require "cucumber" RSpec.describe "Cucumber formatter" do extend ConfigurationHelpers + def do_execute + cli.execute!(existing_runtime) + end + include_context "CI mode activated" do let(:integration_name) { :cucumber } let(:integration_options) { {service_name: "jalapenos"} } end + let(:steps_file_id) { rand(1..2**64 - 1) } + + before do + # Ruby loads any file at most once per process, but we need to load + # the cucumber step definitions multiple times for every Cucumber::Runtime we create + # So we add a random number to the file path to force Ruby to load it again + FileUtils.cp( + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb", + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb" + ) + end + + after do + FileUtils.rm("spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb") + end + # Cucumber runtime setup let(:existing_runtime) { Cucumber::Runtime.new(runtime_options) } let(:runtime_options) { {} } @@ -32,15 +53,11 @@ let(:args) do [ "-r", - "spec/datadog/ci/contrib/cucumber/features/step_definitions", + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb", "spec/datadog/ci/contrib/cucumber/features/passing.feature" ] end - def do_execute - cli.execute!(existing_runtime) - end - it "creates spans for each scenario and step" do expect(Datadog::CI::Ext::Environment).to receive(:tags).never @@ -133,4 +150,36 @@ def do_execute expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) end end + + context "executing a failing test suite" do + let(:args) do + [ + "-r", + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb", + "spec/datadog/ci/contrib/cucumber/features/failing.feature" + ] + end + + it "creates scenario span with failed state" do + expect(kernel).to receive(:exit).with(2) + + do_execute + + expect(first_test_span.name).to eq("cucumber failing scenario") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + + # it "creates test span with failed state" do + # expect(kernel).to receive(:exit).with(2) + + # do_execute + + # expect(first_test_span.name).to eq("cucumber scenario") + # expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + # Datadog::CI::Ext::Test::Status::FAIL + # ) + # end + end end From e2f7eeb19f578826bbecb726f5a2ed92d679313e Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 09:56:39 +0100 Subject: [PATCH 05/11] refactor cucumber instrumentation spec --- .../contrib/cucumber/instrumentation_spec.rb | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 1f1244d2..cba442d9 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -15,30 +15,43 @@ def do_execute end let(:steps_file_id) { rand(1..2**64 - 1) } + let(:steps_file_definition_path) { "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb" } + let(:steps_file_for_run_path) do + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb" + end before do # Ruby loads any file at most once per process, but we need to load # the cucumber step definitions multiple times for every Cucumber::Runtime we create # So we add a random number to the file path to force Ruby to load it again FileUtils.cp( - "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb", - "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb" + steps_file_definition_path, + steps_file_for_run_path ) end after do - FileUtils.rm("spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb") + FileUtils.rm(steps_file_for_run_path) end # Cucumber runtime setup let(:existing_runtime) { Cucumber::Runtime.new(runtime_options) } let(:runtime_options) { {} } # CLI configuration - let(:args) { [] } + let(:feature_file_to_run) {} + let(:features_path) { "spec/datadog/ci/contrib/cucumber/features/#{feature_file_to_run}" } + let(:args) do + [ + "-r", + steps_file_for_run_path, + features_path + ] + end let(:stdin) { StringIO.new } let(:stdout) { StringIO.new } let(:stderr) { StringIO.new } let(:kernel) { double(:kernel) } + let(:cli) do cucumber_8 = Gem::Version.new("8.0.0") @@ -50,13 +63,7 @@ def do_execute end context "executing a passing test suite" do - let(:args) do - [ - "-r", - "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb", - "spec/datadog/ci/contrib/cucumber/features/passing.feature" - ] - end + let(:feature_file_to_run) { "passing.feature" } it "creates spans for each scenario and step" do expect(Datadog::CI::Ext::Environment).to receive(:tags).never @@ -152,13 +159,7 @@ def do_execute end context "executing a failing test suite" do - let(:args) do - [ - "-r", - "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb", - "spec/datadog/ci/contrib/cucumber/features/failing.feature" - ] - end + let(:feature_file_to_run) { "failing.feature" } it "creates scenario span with failed state" do expect(kernel).to receive(:exit).with(2) From 970aa470de05b6294242339f26c8a4fac17079c5 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 10:54:27 +0100 Subject: [PATCH 06/11] track test failures count for Cucumber to set test session status --- lib/datadog/ci/contrib/cucumber/formatter.rb | 75 ++++++++++--------- sig/datadog/ci/contrib/cucumber/formatter.rbs | 5 ++ .../contrib/cucumber/instrumentation_spec.rb | 29 +++++-- vendor/rbs/cucumber/0/cucumber.rbs | 13 ++++ 4 files changed, 78 insertions(+), 44 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 7f2bf6f5..f4e65dcb 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -14,6 +14,7 @@ class Formatter def initialize(config) @config = config + @failed_tests_count = 0 bind_events(config) end @@ -40,25 +41,10 @@ def on_test_run_started(event) end def on_test_run_finished(event) - test_session = CI.active_test_session - test_module = CI.active_test_module - - if test_session && test_module - if event.respond_to?(:success) - if event.success - test_module.passed! - test_session.passed! - else - test_module.failed! - test_session.failed! - end - else - # we need to track results manually if we are using cucumber < 8.0 - test_module.passed! - test_session.passed! - end - test_module.finish - test_session.finish + if event.respond_to?(:success) + finish_session(event.success) + else + finish_session(@failed_tests_count.zero?) end end @@ -79,15 +65,11 @@ def on_test_case_finished(event) test_span = CI.active_test return if test_span.nil? - if event.result.skipped? - test_span.skipped! - elsif event.result.ok? - test_span.passed! - elsif event.result.failed? - test_span.failed! - end + # we need to track test results manually if we are using cucumber < 8.0 because + # TestRunFinished event does not have a success attribute before 8.0 + @failed_tests_count += 1 if event.result.failed? - test_span.finish + finish_test(test_span, event.result) end def on_test_step_started(event) @@ -98,19 +80,40 @@ def on_test_step_finished(event) current_step_span = CI.active_span(Ext::STEP_SPAN_TYPE) return if current_step_span.nil? - if event.result.skipped? - current_step_span.skipped! - elsif event.result.ok? - current_step_span.passed! - elsif event.result.failed? - current_step_span.failed!(exception: event.result.exception) - end - - current_step_span.finish + finish_test(current_step_span, event.result) end private + def finish_test(span, result) + if result.skipped? + span.skipped! + elsif result.ok? + span.passed! + elsif result.failed? + span.failed!(exception: result.exception) + end + span.finish + end + + def finish_session(result) + test_session = CI.active_test_session + test_module = CI.active_test_module + + return unless test_session && test_module + + if result + test_module.passed! + test_session.passed! + else + test_module.failed! + test_session.failed! + end + + test_module.finish + test_session.finish + end + def configuration Datadog.configuration.ci[:cucumber] end diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 46e2e712..b2afe7c7 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -4,6 +4,7 @@ module Datadog module Cucumber class Formatter private + @failed_tests_count: Integer attr_reader config: untyped @@ -27,6 +28,10 @@ module Datadog private + def finish_session: (bool result) -> void + + def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void + def configuration: () -> untyped end end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index cba442d9..61980ced 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -172,15 +172,28 @@ def do_execute ) end - # it "creates test span with failed state" do - # expect(kernel).to receive(:exit).with(2) + it "creates step span with failed state" do + expect(kernel).to receive(:exit).with(2) + + do_execute + + step_span = spans.find { |s| s.resource == "failure" } + expect(step_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end - # do_execute + it "creates test session and test module spans with failed state" do + expect(kernel).to receive(:exit).with(2) - # expect(first_test_span.name).to eq("cucumber scenario") - # expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( - # Datadog::CI::Ext::Test::Status::FAIL - # ) - # end + do_execute + + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end end end diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index c0105432..6dbead93 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -1,6 +1,19 @@ module Cucumber end +module Cucumber::Core +end + +module Cucumber::Core::Test +end + class Cucumber::Runtime def formatters: () -> untyped end + +class Cucumber::Core::Test::Result + def failed?: () -> bool + def ok?: () -> bool + def skipped?: () -> bool + def exception: () -> untyped +end \ No newline at end of file From 9bb93ffef9c9afa0478521d26df8d7fbfa0548f9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 12:28:00 +0100 Subject: [PATCH 07/11] support test suite failed status in cucumber --- lib/datadog/ci/contrib/cucumber/formatter.rb | 46 +++++++++++++++++-- sig/datadog/ci/contrib/cucumber/formatter.rbs | 8 ++++ .../contrib/cucumber/instrumentation_spec.rb | 43 ++++++++++++----- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index f4e65dcb..b01e0c19 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -16,6 +16,9 @@ def initialize(config) @config = config @failed_tests_count = 0 + @current_test_suite = nil + @failed_tests_per_test_suite = Hash.new(0) + bind_events(config) end @@ -49,9 +52,13 @@ def on_test_run_finished(event) end def on_test_case_started(event) + test_suite_name = event.test_case.location.file + + start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name) + CI.start_test( event.test_case.name, - event.test_case.location.file, + test_suite_name, tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, @@ -65,9 +72,15 @@ def on_test_case_finished(event) test_span = CI.active_test return if test_span.nil? - # we need to track test results manually if we are using cucumber < 8.0 because + # we need to track overall test failures manually if we are using cucumber < 8.0 because # TestRunFinished event does not have a success attribute before 8.0 - @failed_tests_count += 1 if event.result.failed? + # In order to keep information about whether test suite failed or passed we need to + # track the number of failed tests per test suite. + if event.result.failed? + @failed_tests_count += 1 + test_suite = @current_test_suite + @failed_tests_per_test_suite[test_suite.name] += 1 if test_suite + end finish_test(test_span, event.result) end @@ -97,6 +110,8 @@ def finish_test(span, result) end def finish_session(result) + finish_current_test_suite + test_session = CI.active_test_session test_module = CI.active_test_module @@ -114,6 +129,31 @@ def finish_session(result) test_session.finish end + def start_test_suite(test_suite_name) + finish_current_test_suite + + @current_test_suite = CI.start_test_suite(test_suite_name) + end + + def finish_current_test_suite + test_suite = @current_test_suite + return unless test_suite + + if @failed_tests_per_test_suite[test_suite.name].zero? + test_suite.passed! + else + test_suite.failed! + end + test_suite.finish + end + + def same_test_suite_as_current?(test_suite_name) + test_suite = @current_test_suite + return false unless test_suite + + test_suite.name == test_suite_name + end + def configuration Datadog.configuration.ci[:cucumber] end diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index b2afe7c7..ef1f133a 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -5,6 +5,8 @@ module Datadog class Formatter private @failed_tests_count: Integer + @current_test_suite: Datadog::CI::Span? + @failed_tests_per_test_suite: Hash[String, Integer] attr_reader config: untyped @@ -28,6 +30,12 @@ module Datadog private + def start_test_suite: (String test_suite_name) -> void + + def finish_current_test_suite: () -> void + + def same_test_suite_as_current?: (String test_suite_name) -> bool + def finish_session: (bool result) -> void def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 61980ced..fa655768 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -147,6 +147,29 @@ def do_execute expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) end + it "creates test suite span" do + expect(kernel).to receive(:exit).with(0) + + do_execute + + expect(test_suite_span).not_to be_nil + expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.service).to eq("jalapenos") + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + end + it "connects scenario span to test session and test module" do expect(kernel).to receive(:exit).with(0) @@ -155,13 +178,15 @@ def do_execute expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_command) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(test_suite_span.name) end end context "executing a failing test suite" do let(:feature_file_to_run) { "failing.feature" } - it "creates scenario span with failed state" do + it "creates all CI spans with failed state" do expect(kernel).to receive(:exit).with(2) do_execute @@ -170,23 +195,17 @@ def do_execute expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL ) - end - - it "creates step span with failed state" do - expect(kernel).to receive(:exit).with(2) - - do_execute step_span = spans.find { |s| s.resource == "failure" } + expect(step_span.name).to eq("failure") expect(step_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL ) - end - - it "creates test session and test module spans with failed state" do - expect(kernel).to receive(:exit).with(2) - do_execute + expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL From 4792e017291f8185274d8c14936de85ef2e8a066 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 12:57:05 +0100 Subject: [PATCH 08/11] test several feature files together in cucumber instrumentation spec --- lib/datadog/ci/contrib/cucumber/formatter.rb | 7 +- .../contrib/cucumber/instrumentation_spec.rb | 107 +++++++++++------- spec/support/tracer_helpers.rb | 6 +- 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index b01e0c19..6cb67cab 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -72,9 +72,10 @@ def on_test_case_finished(event) test_span = CI.active_test return if test_span.nil? - # we need to track overall test failures manually if we are using cucumber < 8.0 because - # TestRunFinished event does not have a success attribute before 8.0 - # In order to keep information about whether test suite failed or passed we need to + # We need to track overall test failures manually if we are using cucumber < 8.0 because + # TestRunFinished event does not have a success attribute before 8.0. + # + # To track whether test suite failed or passed we need to # track the number of failed tests per test suite. if event.result.failed? @failed_tests_count += 1 diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index fa655768..30209b8d 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -5,10 +5,6 @@ RSpec.describe "Cucumber formatter" do extend ConfigurationHelpers - def do_execute - cli.execute!(existing_runtime) - end - include_context "CI mode activated" do let(:integration_name) { :cucumber } let(:integration_options) { {service_name: "jalapenos"} } @@ -20,20 +16,6 @@ def do_execute "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb" end - before do - # Ruby loads any file at most once per process, but we need to load - # the cucumber step definitions multiple times for every Cucumber::Runtime we create - # So we add a random number to the file path to force Ruby to load it again - FileUtils.cp( - steps_file_definition_path, - steps_file_for_run_path - ) - end - - after do - FileUtils.rm(steps_file_for_run_path) - end - # Cucumber runtime setup let(:existing_runtime) { Cucumber::Runtime.new(runtime_options) } let(:runtime_options) { {} } @@ -62,16 +44,34 @@ def do_execute end end - context "executing a passing test suite" do - let(:feature_file_to_run) { "passing.feature" } + def run_cucumber_tests + cli.execute!(existing_runtime) + end - it "creates spans for each scenario and step" do - expect(Datadog::CI::Ext::Environment).to receive(:tags).never + let(:expected_test_run_code) { 0 } - expect(kernel).to receive(:exit).with(0) + before do + # Ruby loads any file at most once per process, but we need to load + # the cucumber step definitions multiple times for every Cucumber::Runtime we create + # So we add a random number to the file path to force Ruby to load it again + FileUtils.cp( + steps_file_definition_path, + steps_file_for_run_path + ) + + expect(Datadog::CI::Ext::Environment).to receive(:tags).never + expect(kernel).to receive(:exit).with(expected_test_run_code) + run_cucumber_tests + end + + after do + FileUtils.rm(steps_file_for_run_path) + end - do_execute + context "executing a passing test suite" do + let(:feature_file_to_run) { "passing.feature" } + it "creates spans for each scenario and step" do scenario_span = spans.find { |s| s.resource == "cucumber scenario" } expect(scenario_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) @@ -103,10 +103,6 @@ def do_execute end it "creates test session span" do - expect(kernel).to receive(:exit).with(0) - - do_execute - expect(test_session_span).not_to be_nil expect(test_session_span.service).to eq("jalapenos") expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( @@ -125,10 +121,6 @@ def do_execute end it "creates test module span" do - expect(kernel).to receive(:exit).with(0) - - do_execute - expect(test_module_span).not_to be_nil expect(test_module_span.name).to eq(test_command) expect(test_module_span.service).to eq("jalapenos") @@ -148,10 +140,6 @@ def do_execute end it "creates test suite span" do - expect(kernel).to receive(:exit).with(0) - - do_execute - expect(test_suite_span).not_to be_nil expect(test_suite_span.name).to eq(features_path) expect(test_suite_span.service).to eq("jalapenos") @@ -171,10 +159,6 @@ def do_execute end it "connects scenario span to test session and test module" do - expect(kernel).to receive(:exit).with(0) - - do_execute - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_command) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) @@ -185,12 +169,9 @@ def do_execute context "executing a failing test suite" do let(:feature_file_to_run) { "failing.feature" } + let(:expected_test_run_code) { 2 } it "creates all CI spans with failed state" do - expect(kernel).to receive(:exit).with(2) - - do_execute - expect(first_test_span.name).to eq("cucumber failing scenario") expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL @@ -215,4 +196,42 @@ def do_execute ) end end + + context "executing several features at once" do + let(:expected_test_run_code) { 2 } + + let(:passing_test_suite) { test_suite_spans.find { |span| span.name =~ /passing/ } } + let(:failing_test_suite) { test_suite_spans.find { |span| span.name =~ /failing/ } } + + it "creates a test suite span for each feature" do + expect(test_suite_spans.count).to eq(2) + expect(passing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + expect(failing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + + it "connects tests with their respective test suites" do + cucumber_scenario = test_spans.find { |span| span.name =~ /cucumber scenario/ } + expect(cucumber_scenario.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq( + passing_test_suite.id.to_s + ) + + cucumber_failing_scenario = test_spans.find { |span| span.name =~ /cucumber failing scenario/ } + expect(cucumber_failing_scenario.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq( + failing_test_suite.id.to_s + ) + end + + it "sets failed status for module and session" do + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + end end diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index ff1460af..c9022a1e 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -104,7 +104,7 @@ def test_module_span end def test_suite_span - spans.find { |span| span.type == "test_suite_end" } + test_suite_spans.first end def first_test_span @@ -119,6 +119,10 @@ def test_spans spans.filter { |span| span.type == "test" } end + def test_suite_spans + spans.filter { |span| span.type == "test_suite_end" } + end + def tracer_spans spans.filter { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) } end From 31620e18820a9d1ebd38494fc96537a522b85ce0 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 12:59:38 +0100 Subject: [PATCH 09/11] rename steps_file_id to run_id to better communicate its purpose --- spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 30209b8d..75e088ab 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -10,10 +10,10 @@ let(:integration_options) { {service_name: "jalapenos"} } end - let(:steps_file_id) { rand(1..2**64 - 1) } + let(:run_id) { rand(1..2**64 - 1) } let(:steps_file_definition_path) { "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb" } let(:steps_file_for_run_path) do - "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{steps_file_id}.rb" + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{run_id}.rb" end # Cucumber runtime setup From c65de65467e9ceff6b640a669ef2c8a16fdb0091 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 14 Dec 2023 13:04:23 +0100 Subject: [PATCH 10/11] inline function run_cucumber_tests in cucumber instrumentation specs as it is called once --- spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 75e088ab..33e93344 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -44,10 +44,6 @@ end end - def run_cucumber_tests - cli.execute!(existing_runtime) - end - let(:expected_test_run_code) { 0 } before do @@ -61,7 +57,8 @@ def run_cucumber_tests expect(Datadog::CI::Ext::Environment).to receive(:tags).never expect(kernel).to receive(:exit).with(expected_test_run_code) - run_cucumber_tests + + cli.execute!(existing_runtime) end after do From 0bdbaa128fd74aae717a67b37de26f683c5c05fe Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 15 Dec 2023 10:15:36 +0100 Subject: [PATCH 11/11] fix potential performance issue when storing all the failure counts per test suite - for sequential execution one integer is enough --- lib/datadog/ci/contrib/cucumber/formatter.rb | 10 +++++----- sig/datadog/ci/contrib/cucumber/formatter.rbs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 6cb67cab..9af75fb4 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -17,7 +17,7 @@ def initialize(config) @failed_tests_count = 0 @current_test_suite = nil - @failed_tests_per_test_suite = Hash.new(0) + @failed_tests_in_current_test_suite = 0 bind_events(config) end @@ -76,11 +76,10 @@ def on_test_case_finished(event) # TestRunFinished event does not have a success attribute before 8.0. # # To track whether test suite failed or passed we need to - # track the number of failed tests per test suite. + # track the number of failed tests in the current test suite. if event.result.failed? @failed_tests_count += 1 - test_suite = @current_test_suite - @failed_tests_per_test_suite[test_suite.name] += 1 if test_suite + @failed_tests_in_current_test_suite += 1 end finish_test(test_span, event.result) @@ -140,11 +139,12 @@ def finish_current_test_suite test_suite = @current_test_suite return unless test_suite - if @failed_tests_per_test_suite[test_suite.name].zero? + if @failed_tests_in_current_test_suite.zero? test_suite.passed! else test_suite.failed! end + @failed_tests_in_current_test_suite = 0 test_suite.finish end diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index ef1f133a..9f2c1d5c 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -6,7 +6,7 @@ module Datadog private @failed_tests_count: Integer @current_test_suite: Datadog::CI::Span? - @failed_tests_per_test_suite: Hash[String, Integer] + @failed_tests_in_current_test_suite: Integer attr_reader config: untyped