diff --git a/lib/datadog/ci/contrib/cucumber/configuration/settings.rb b/lib/datadog/ci/contrib/cucumber/configuration/settings.rb index 3e916f11..e99a4948 100644 --- a/lib/datadog/ci/contrib/cucumber/configuration/settings.rb +++ b/lib/datadog/ci/contrib/cucumber/configuration/settings.rb @@ -21,7 +21,7 @@ class Settings < Datadog::CI::Contrib::Settings option :service_name do |o| o.type :string - o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME } + o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME } end # @deprecated Will be removed in 1.0 diff --git a/lib/datadog/ci/contrib/cucumber/ext.rb b/lib/datadog/ci/contrib/cucumber/ext.rb index 4878a445..f8043a1f 100644 --- a/lib/datadog/ci/contrib/cucumber/ext.rb +++ b/lib/datadog/ci/contrib/cucumber/ext.rb @@ -7,14 +7,16 @@ module Cucumber # Cucumber integration constants # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes. module Ext - APP = "cucumber" ENV_ENABLED = "DD_TRACE_CUCUMBER_ENABLED" - ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME" + DEFAULT_SERVICE_NAME = "cucumber" + FRAMEWORK = "cucumber" - OPERATION_NAME = "cucumber.test" - SERVICE_NAME = "cucumber" + STEP_SPAN_TYPE = "step" - TEST_TYPE = "test" + + # TODO: remove in 1.0 + ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME" + OPERATION_NAME = "cucumber.test" end end end diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 2a2f3e34..9af75fb4 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -9,31 +9,60 @@ module Contrib module Cucumber # Defines collection of instrumented Cucumber events class Formatter - attr_reader :config, :current_feature_span, :current_step_span + attr_reader :config private :config - private :current_feature_span, :current_step_span def initialize(config) @config = config + @failed_tests_count = 0 + + @current_test_suite = nil + @failed_tests_in_current_test_suite = 0 bind_events(config) end def bind_events(config) + config.on_event :test_run_started, &method(:on_test_run_started) + config.on_event :test_run_finished, &method(:on_test_run_finished) config.on_event :test_case_started, &method(:on_test_case_started) config.on_event :test_case_finished, &method(:on_test_case_finished) config.on_event :test_step_started, &method(:on_test_step_started) config.on_event :test_step_finished, &method(:on_test_step_finished) end + def on_test_run_started(event) + test_session = CI.start_test_session( + tags: { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE + }, + service: configuration[:service_name] + ) + CI.start_test_module(test_session.name) + end + + def on_test_run_finished(event) + if event.respond_to?(:success) + finish_session(event.success) + else + finish_session(@failed_tests_count.zero?) + end + end + def on_test_case_started(event) + test_suite_name = event.test_case.location.file + + start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name) + CI.start_test( event.test_case.name, - event.test_case.location.file, + test_suite_name, tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE }, service: configuration[:service_name] ) @@ -43,15 +72,17 @@ def on_test_case_finished(event) test_span = CI.active_test return if test_span.nil? - if event.result.skipped? - test_span.skipped! - elsif event.result.ok? - test_span.passed! - elsif event.result.failed? - test_span.failed! + # We need to track overall test failures manually if we are using cucumber < 8.0 because + # TestRunFinished event does not have a success attribute before 8.0. + # + # To track whether test suite failed or passed we need to + # track the number of failed tests in the current test suite. + if event.result.failed? + @failed_tests_count += 1 + @failed_tests_in_current_test_suite += 1 end - test_span.finish + finish_test(test_span, event.result) end def on_test_step_started(event) @@ -62,18 +93,67 @@ def on_test_step_finished(event) current_step_span = CI.active_span(Ext::STEP_SPAN_TYPE) return if current_step_span.nil? - if event.result.skipped? - current_step_span.skipped! - elsif event.result.ok? - current_step_span.passed! - elsif event.result.failed? - current_step_span.failed!(exception: event.result.exception) + finish_test(current_step_span, event.result) + end + + private + + def finish_test(span, result) + if result.skipped? + span.skipped! + elsif result.ok? + span.passed! + elsif result.failed? + span.failed!(exception: result.exception) end + span.finish + end + + def finish_session(result) + finish_current_test_suite + + test_session = CI.active_test_session + test_module = CI.active_test_module + + return unless test_session && test_module - current_step_span.finish + if result + test_module.passed! + test_session.passed! + else + test_module.failed! + test_session.failed! + end + + test_module.finish + test_session.finish end - private + def start_test_suite(test_suite_name) + finish_current_test_suite + + @current_test_suite = CI.start_test_suite(test_suite_name) + end + + def finish_current_test_suite + test_suite = @current_test_suite + return unless test_suite + + if @failed_tests_in_current_test_suite.zero? + test_suite.passed! + else + test_suite.failed! + end + @failed_tests_in_current_test_suite = 0 + test_suite.finish + end + + def same_test_suite_as_current?(test_suite_name) + test_suite = @current_test_suite + return false unless test_suite + + test_suite.name == test_suite_name + end def configuration Datadog.configuration.ci[:cucumber] diff --git a/sig/datadog/ci/contrib/cucumber/ext.rbs b/sig/datadog/ci/contrib/cucumber/ext.rbs index 112bcda7..18d81294 100644 --- a/sig/datadog/ci/contrib/cucumber/ext.rbs +++ b/sig/datadog/ci/contrib/cucumber/ext.rbs @@ -3,8 +3,6 @@ module Datadog module Contrib module Cucumber module Ext - APP: String - ENV_ENABLED: String ENV_OPERATION_NAME: String @@ -13,11 +11,9 @@ module Datadog OPERATION_NAME: String - SERVICE_NAME: String + DEFAULT_SERVICE_NAME: String STEP_SPAN_TYPE: String - - TEST_TYPE: String end end end diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 30143e0a..9f2c1d5c 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -4,19 +4,22 @@ module Datadog module Cucumber class Formatter private + @failed_tests_count: Integer + @current_test_suite: Datadog::CI::Span? + @failed_tests_in_current_test_suite: Integer attr_reader config: untyped - attr_reader current_feature_span: untyped - - attr_reader current_step_span: untyped - public def initialize: (untyped config) -> void def bind_events: (untyped config) -> untyped + def on_test_run_started: (untyped event) -> untyped + + def on_test_run_finished: (untyped event) -> untyped + def on_test_case_started: (untyped event) -> untyped def on_test_case_finished: (untyped event) -> (nil | untyped) @@ -27,6 +30,16 @@ module Datadog private + def start_test_suite: (String test_suite_name) -> void + + def finish_current_test_suite: () -> void + + def same_test_suite_as_current?: (String test_suite_name) -> bool + + def finish_session: (bool result) -> void + + def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void + def configuration: () -> untyped end end diff --git a/spec/datadog/ci/contrib/cucumber/features/failing.feature b/spec/datadog/ci/contrib/cucumber/features/failing.feature new file mode 100644 index 00000000..518b6ed3 --- /dev/null +++ b/spec/datadog/ci/contrib/cucumber/features/failing.feature @@ -0,0 +1,5 @@ +Feature: Datadog integration + Scenario: cucumber failing scenario + Given datadog + And datadog + Then failure diff --git a/spec/datadog/ci/contrib/cucumber/cucumber.features b/spec/datadog/ci/contrib/cucumber/features/passing.feature similarity index 70% rename from spec/datadog/ci/contrib/cucumber/cucumber.features rename to spec/datadog/ci/contrib/cucumber/features/passing.feature index 4b5209c9..38be690a 100644 --- a/spec/datadog/ci/contrib/cucumber/cucumber.features +++ b/spec/datadog/ci/contrib/cucumber/features/passing.feature @@ -2,3 +2,5 @@ Feature: Datadog integration Scenario: cucumber scenario Given datadog + And datadog + Then datadog \ No newline at end of file diff --git a/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb new file mode 100644 index 00000000..b56a3d96 --- /dev/null +++ b/spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb @@ -0,0 +1,7 @@ +Then "datadog" do + true +end + +Then "failure" do + expect(1 + 1).to eq(3) +end diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 8ad3a25a..33e93344 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -1,4 +1,5 @@ require "stringio" +require "fileutils" require "cucumber" RSpec.describe "Cucumber formatter" do @@ -9,15 +10,30 @@ let(:integration_options) { {service_name: "jalapenos"} } end + let(:run_id) { rand(1..2**64 - 1) } + let(:steps_file_definition_path) { "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb" } + let(:steps_file_for_run_path) do + "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps_#{run_id}.rb" + end + # Cucumber runtime setup let(:existing_runtime) { Cucumber::Runtime.new(runtime_options) } let(:runtime_options) { {} } # CLI configuration - let(:args) { [] } + let(:feature_file_to_run) {} + let(:features_path) { "spec/datadog/ci/contrib/cucumber/features/#{feature_file_to_run}" } + let(:args) do + [ + "-r", + steps_file_for_run_path, + features_path + ] + end let(:stdin) { StringIO.new } let(:stdout) { StringIO.new } let(:stderr) { StringIO.new } let(:kernel) { double(:kernel) } + let(:cli) do cucumber_8 = Gem::Version.new("8.0.0") @@ -28,27 +44,53 @@ end end - context "executing a test suite" do - let(:args) { ["spec/datadog/ci/contrib/cucumber/cucumber.features"] } + let(:expected_test_run_code) { 0 } - def do_execute - cli.execute!(existing_runtime) - end + before do + # Ruby loads any file at most once per process, but we need to load + # the cucumber step definitions multiple times for every Cucumber::Runtime we create + # So we add a random number to the file path to force Ruby to load it again + FileUtils.cp( + steps_file_definition_path, + steps_file_for_run_path + ) - it "creates spans for each scenario and step" do - expect(Datadog::CI::Ext::Environment).to receive(:tags).never + expect(Datadog::CI::Ext::Environment).to receive(:tags).never + expect(kernel).to receive(:exit).with(expected_test_run_code) - expect(kernel).to receive(:exit).with(0) + cli.execute!(existing_runtime) + end + + after do + FileUtils.rm(steps_file_for_run_path) + end - do_execute + context "executing a passing test suite" do + let(:feature_file_to_run) { "passing.feature" } + it "creates spans for each scenario and step" do scenario_span = spans.find { |s| s.resource == "cucumber scenario" } - step_span = spans.find { |s| s.resource == "datadog" } - expect(scenario_span.resource).to eq("cucumber scenario") - expect(scenario_span.service).to eq("jalapenos") expect(scenario_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) expect(scenario_span.name).to eq("cucumber scenario") + expect(scenario_span.resource).to eq("cucumber scenario") + expect(scenario_span.service).to eq("jalapenos") + + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("cucumber scenario") + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( + "spec/datadog/ci/contrib/cucumber/features/passing.feature" + ) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + + step_span = spans.find { |s| s.resource == "datadog" } expect(step_span.resource).to eq("datadog") spans.each do |span| @@ -56,5 +98,137 @@ def do_execute .to eq(Datadog::CI::Ext::Test::CONTEXT_ORIGIN) end end + + it "creates test session span" do + expect(test_session_span).not_to be_nil + expect(test_session_span.service).to eq("jalapenos") + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + end + + it "creates test module span" do + expect(test_module_span).not_to be_nil + expect(test_module_span.name).to eq(test_command) + expect(test_module_span.service).to eq("jalapenos") + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + end + + it "creates test suite span" do + expect(test_suite_span).not_to be_nil + expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.service).to eq("jalapenos") + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Cucumber::Integration.version.to_s + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS) + end + + it "connects scenario span to test session and test module" do + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID)).to eq(test_module_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_MODULE)).to eq(test_command) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID)).to eq(test_session_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(test_suite_span.name) + end + end + + context "executing a failing test suite" do + let(:feature_file_to_run) { "failing.feature" } + let(:expected_test_run_code) { 2 } + + it "creates all CI spans with failed state" do + expect(first_test_span.name).to eq("cucumber failing scenario") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + + step_span = spans.find { |s| s.resource == "failure" } + expect(step_span.name).to eq("failure") + expect(step_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + + expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + end + + context "executing several features at once" do + let(:expected_test_run_code) { 2 } + + let(:passing_test_suite) { test_suite_spans.find { |span| span.name =~ /passing/ } } + let(:failing_test_suite) { test_suite_spans.find { |span| span.name =~ /failing/ } } + + it "creates a test suite span for each feature" do + expect(test_suite_spans.count).to eq(2) + expect(passing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + expect(failing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + + it "connects tests with their respective test suites" do + cucumber_scenario = test_spans.find { |span| span.name =~ /cucumber scenario/ } + expect(cucumber_scenario.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq( + passing_test_suite.id.to_s + ) + + cucumber_failing_scenario = test_spans.find { |span| span.name =~ /cucumber failing scenario/ } + expect(cucumber_failing_scenario.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq( + failing_test_suite.id.to_s + ) + end + + it "sets failed status for module and session" do + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end end end diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index ff1460af..c9022a1e 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -104,7 +104,7 @@ def test_module_span end def test_suite_span - spans.find { |span| span.type == "test_suite_end" } + test_suite_spans.first end def first_test_span @@ -119,6 +119,10 @@ def test_spans spans.filter { |span| span.type == "test" } end + def test_suite_spans + spans.filter { |span| span.type == "test_suite_end" } + end + def tracer_spans spans.filter { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) } end diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index c0105432..6dbead93 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -1,6 +1,19 @@ module Cucumber end +module Cucumber::Core +end + +module Cucumber::Core::Test +end + class Cucumber::Runtime def formatters: () -> untyped end + +class Cucumber::Core::Test::Result + def failed?: () -> bool + def ok?: () -> bool + def skipped?: () -> bool + def exception: () -> untyped +end \ No newline at end of file