diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb index 81a21273..7689a988 100644 --- a/lib/datadog/ci/contrib/cucumber/formatter.rb +++ b/lib/datadog/ci/contrib/cucumber/formatter.rb @@ -54,7 +54,7 @@ def on_test_run_finished(event) end def on_test_case_started(event) - test_suite_name = event.test_case.location.file + test_suite_name = test_suite_name(event.test_case) tags = { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, @@ -108,6 +108,21 @@ def on_test_step_finished(event) private + def test_suite_name(test_case) + feature = if test_case.respond_to?(:feature) + test_case.feature + elsif @ast_lookup + gherkin_doc = @ast_lookup.gherkin_document(test_case.location.file) + gherkin_doc.feature if gherkin_doc + end + + if feature + "#{feature.name} at #{test_case.location.file}" + else + test_case.location.file + end + end + def finish_test(span, result) if result.skipped? span.skipped! diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb index 84a4c07f..c635c0ab 100644 --- a/lib/datadog/ci/contrib/minitest/hooks.rb +++ b/lib/datadog/ci/contrib/minitest/hooks.rb @@ -14,8 +14,6 @@ def before_setup super return unless datadog_configuration[:enabled] - test_name = "#{class_name}##{name}" - test_suite_name = Helpers.test_suite_name(self.class, name) if Helpers.parallel?(self.class) test_suite_name = "#{test_suite_name} (#{name} concurrently)" @@ -27,7 +25,7 @@ def before_setup source_file, line_number = method(name).source_location CI.start_test( - test_name, + name, test_suite_name, tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb index a23677cb..3014d758 100644 --- a/lib/datadog/ci/contrib/rspec/example.rb +++ b/lib/datadog/ci/contrib/rspec/example.rb @@ -14,10 +14,9 @@ def self.included(base) base.prepend(InstanceMethods) end - # Instance methods for configuration module InstanceMethods - def run(example_group_instance, reporter) - return super unless configuration[:enabled] + def run(*) + return super unless datadog_configuration[:enabled] test_name = full_description.strip if metadata[:description].empty? @@ -25,9 +24,15 @@ def run(example_group_instance, reporter) test_name += " #{description}" end + test_suite_description = fetch_top_level_example_group[:description] + suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}" + + # remove suite name from test name to avoid duplication + test_name = test_name.sub(test_suite_description, "").strip + CI.trace_test( test_name, - metadata[:example_group][:rerun_file_path], + suite_name, tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s, @@ -35,7 +40,7 @@ def run(example_group_instance, reporter) CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]), CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s }, - service: configuration[:service_name] + service: datadog_configuration[:service_name] ) do |test_span| test_span.set_parameters({}, {"scoped_id" => metadata[:scoped_id]}) @@ -56,7 +61,17 @@ def run(example_group_instance, reporter) private - def configuration + def fetch_top_level_example_group + return metadata[:example_group] unless metadata[:example_group][:parent_example_group] + + res = metadata[:example_group][:parent_example_group] + while (parent = res[:parent_example_group]) + res = parent + end + res + end + + def datadog_configuration Datadog.configuration.ci[:rspec] end end diff --git a/lib/datadog/ci/contrib/rspec/example_group.rb b/lib/datadog/ci/contrib/rspec/example_group.rb index 52151513..cd3725e9 100644 --- a/lib/datadog/ci/contrib/rspec/example_group.rb +++ b/lib/datadog/ci/contrib/rspec/example_group.rb @@ -7,7 +7,7 @@ module Datadog module CI module Contrib module RSpec - # Instrument RSpec::Core::Example + # Instrument RSpec::Core::ExampleGroup module ExampleGroup def self.included(base) base.singleton_class.prepend(ClassMethods) @@ -15,11 +15,12 @@ def self.included(base) # Instance methods for configuration module ClassMethods - def run(reporter = ::RSpec::Core::NullReporter) - return super unless configuration[:enabled] + def run(*) + return super unless datadog_configuration[:enabled] return super unless top_level? - test_suite = Datadog::CI.start_test_suite(file_path) + suite_name = "#{description} at #{file_path}" + test_suite = Datadog::CI.start_test_suite(suite_name) result = super @@ -35,7 +36,7 @@ def run(reporter = ::RSpec::Core::NullReporter) private - def configuration + def datadog_configuration Datadog.configuration.ci[:rspec] end end diff --git a/lib/datadog/ci/contrib/rspec/runner.rb b/lib/datadog/ci/contrib/rspec/runner.rb index e2473615..1ce11dd5 100644 --- a/lib/datadog/ci/contrib/rspec/runner.rb +++ b/lib/datadog/ci/contrib/rspec/runner.rb @@ -14,8 +14,8 @@ def self.included(base) end module InstanceMethods - def run_specs(example_groups) - return super unless configuration[:enabled] + def run_specs(*) + return super unless datadog_configuration[:enabled] test_session = CI.start_test_session( tags: { @@ -23,7 +23,7 @@ def run_specs(example_groups) CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s, CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE }, - service: configuration[:service_name] + service: datadog_configuration[:service_name] ) test_module = CI.start_test_module(test_session.name) @@ -46,7 +46,7 @@ def run_specs(example_groups) private - def configuration + def datadog_configuration Datadog.configuration.ci[:rspec] end end diff --git a/sig/datadog/ci/contrib/cucumber/formatter.rbs b/sig/datadog/ci/contrib/cucumber/formatter.rbs index 216e7569..523246ae 100644 --- a/sig/datadog/ci/contrib/cucumber/formatter.rbs +++ b/sig/datadog/ci/contrib/cucumber/formatter.rbs @@ -30,6 +30,8 @@ module Datadog private + def test_suite_name: (untyped test_case) -> String + def start_test_suite: (String test_suite_name) -> void def finish_current_test_suite: () -> void diff --git a/sig/datadog/ci/contrib/rspec/example.rbs b/sig/datadog/ci/contrib/rspec/example.rbs index 372e35e2..5acf73bd 100644 --- a/sig/datadog/ci/contrib/rspec/example.rbs +++ b/sig/datadog/ci/contrib/rspec/example.rbs @@ -11,7 +11,8 @@ module Datadog private - def configuration: () -> untyped + def fetch_top_level_example_group: () -> Hash[Symbol, untyped] + def datadog_configuration: () -> untyped end end end diff --git a/sig/datadog/ci/contrib/rspec/example_group.rbs b/sig/datadog/ci/contrib/rspec/example_group.rbs index 9daddd63..dec0f3f2 100644 --- a/sig/datadog/ci/contrib/rspec/example_group.rbs +++ b/sig/datadog/ci/contrib/rspec/example_group.rbs @@ -12,7 +12,7 @@ module Datadog private - def configuration: () -> untyped + def datadog_configuration: () -> untyped end end end diff --git a/sig/datadog/ci/contrib/rspec/runner.rbs b/sig/datadog/ci/contrib/rspec/runner.rbs index ef12b471..e806c556 100644 --- a/sig/datadog/ci/contrib/rspec/runner.rbs +++ b/sig/datadog/ci/contrib/rspec/runner.rbs @@ -12,7 +12,7 @@ module Datadog private - def configuration: () -> untyped + def datadog_configuration: () -> untyped end end end diff --git a/spec/datadog/ci/contrib/cucumber/features/failing.feature b/spec/datadog/ci/contrib/cucumber/features/failing.feature index 518b6ed3..9ca09aff 100644 --- a/spec/datadog/ci/contrib/cucumber/features/failing.feature +++ b/spec/datadog/ci/contrib/cucumber/features/failing.feature @@ -1,4 +1,4 @@ -Feature: Datadog integration +Feature: Datadog integration - test failing features Scenario: cucumber failing scenario Given datadog And datadog diff --git a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb index 77fdbb7f..220d53c2 100644 --- a/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb @@ -80,7 +80,7 @@ expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("cucumber scenario") expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( - "spec/datadog/ci/contrib/cucumber/features/passing.feature" + "Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature" ) expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE) expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( @@ -147,7 +147,7 @@ it "creates test suite span" do expect(test_suite_span).not_to be_nil - expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.name).to eq("Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature") expect(test_suite_span.service).to eq("jalapenos") expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( Datadog::CI::Ext::AppTypes::TYPE_TEST @@ -189,7 +189,9 @@ Datadog::CI::Ext::Test::Status::FAIL ) - expect(test_suite_span.name).to eq(features_path) + expect(test_suite_span.name).to eq( + "Datadog integration - test failing features at spec/datadog/ci/contrib/cucumber/features/failing.feature" + ) expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL ) @@ -224,7 +226,7 @@ ) end expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( - "spec/datadog/ci/contrib/cucumber/features/with_parameters.feature" + "Datadog integration for parametrized tests at spec/datadog/ci/contrib/cucumber/features/with_parameters.feature" ) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb index 623d0edf..1cc5e773 100644 --- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb @@ -51,10 +51,10 @@ def test_foo klass.new(:test_foo).run expect(span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) - expect(span.name).to eq("SomeTest#test_foo") - expect(span.resource).to eq("SomeTest#test_foo") + expect(span.name).to eq("test_foo") + expect(span.resource).to eq("test_foo") expect(span.service).to eq("ltest") - expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeTest#test_foo") + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("test_foo") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" ) @@ -110,9 +110,9 @@ def self.name klass.new(method_name).run expect(span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) - expect(span.resource).to eq("SomeSpec##{method_name}") + expect(span.resource).to eq(method_name) expect(span.service).to eq("ltest") - expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeSpec##{method_name}") + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq(method_name) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" ) @@ -514,7 +514,7 @@ def test_fail end it "traces test, test session, test module with failed status" do - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeFailedTest#test_fail") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("test_fail") expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( Datadog::CI::Ext::Test::Status::FAIL ) @@ -561,10 +561,10 @@ class SomeSpec < Minitest::Spec expect(test_names).to eq( [ - "SomeSpec#test_0001_does not fail", - "in context#test_0001_does not fail", - "in context::deeper context#test_0001_does not fail", - "in other context#test_0001_does not fail" + "test_0001_does not fail", + "test_0001_does not fail", + "test_0001_does not fail", + "test_0001_does not fail" ] ) end @@ -635,10 +635,10 @@ def test_b_2 test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort expect(test_names).to eq( [ - "TestA#test_a_1", - "TestA#test_a_2", - "TestB#test_b_1", - "TestB#test_b_2" + "test_a_1", + "test_a_2", + "test_b_1", + "test_b_2" ] ) diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index d15902c1..2e38dce0 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -30,11 +30,13 @@ def with_new_rspec_environment end expect(first_test_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) - expect(first_test_span.name).to eq("some test foo") - expect(first_test_span.resource).to eq("some test foo") expect(first_test_span.service).to eq("lspec") - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some test foo") - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path) + + expect(first_test_span.name).to eq("foo") + expect(first_test_span.resource).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq("some test at #{spec.file_path}") expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE) expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK) @@ -76,7 +78,7 @@ def with_new_rspec_environment end.run end - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to match(/some unnamed test example at .+/) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to match(/example at .+/) end it "creates span for deeply nested examples" do @@ -108,9 +110,9 @@ def with_new_rspec_environment end.tap(&:run) end - expect(first_test_span.resource).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo") - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("some nested test 1 2 3 4 5 6 7 8 9 10 foo") - expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path) + expect(first_test_span.resource).to eq("1 2 3 4 5 6 7 8 9 10 foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("1 2 3 4 5 6 7 8 9 10 foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq("some nested test at #{spec.file_path}") end it "creates spans for example with instrumentation" do @@ -234,20 +236,22 @@ def devnull def rspec_session_run(with_failed_test: false, with_shared_test: false) with_new_rspec_environment do spec = RSpec.describe "SomeTest" do - it "foo" do - # DO NOTHING - end + context "nested" do + it "foo" do + # DO NOTHING + end - if with_failed_test - it "fails" do - expect(1).to eq(2) + if with_failed_test + it "fails" do + expect(1).to eq(2) + end end - end - if with_shared_test - require_relative "some_shared_examples" - include_examples "Testing shared examples", 2 - include_examples "Testing shared examples", 1 + if with_shared_test + require_relative "some_shared_examples" + include_examples "Testing shared examples", 2 + include_examples "Testing shared examples", 1 + end end end @@ -312,7 +316,7 @@ def rspec_session_run(with_failed_test: false, with_shared_test: false) expect(test_suite_span).not_to be_nil expect(test_suite_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SUITE) - expect(test_suite_span.name).to eq(spec.file_path) + expect(test_suite_span.name).to eq("SomeTest at #{spec.file_path}") expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( Datadog::CI::Ext::AppTypes::TYPE_TEST @@ -372,14 +376,14 @@ def rspec_session_run(with_failed_test: false, with_shared_test: false) let!(:spec) { rspec_session_run(with_shared_test: true) } it "creates correct test spans connects all tests to a single test suite" do - shared_test_spans = test_spans.filter { |test_span| test_span.name == "SomeTest shared examples adds 1 and 1" } + shared_test_spans = test_spans.filter { |test_span| test_span.name == "nested shared examples adds 1 and 1" } expect(shared_test_spans).to have(2).items shared_test_spans.each_with_index do |shared_test_span, index| - expect(shared_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(spec.file_path) + expect(shared_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq("SomeTest at #{spec.file_path}") expect(shared_test_span.get_tag(Datadog::CI::Ext::Test::TAG_PARAMETERS)).to eq( - "{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:#{2 + index}:1\"}}" + "{\"arguments\":{},\"metadata\":{\"scoped_id\":\"1:1:#{2 + index}:1\"}}" ) end diff --git a/vendor/rbs/cucumber/0/cucumber.rbs b/vendor/rbs/cucumber/0/cucumber.rbs index 45f541ac..6e643260 100644 --- a/vendor/rbs/cucumber/0/cucumber.rbs +++ b/vendor/rbs/cucumber/0/cucumber.rbs @@ -14,7 +14,6 @@ end module Cucumber::Formatter end - class Cucumber::Core::Test::Result def failed?: () -> bool def ok?: () -> bool @@ -26,4 +25,16 @@ class Cucumber::Formatter::AstLookup def initialize: (untyped config) -> void def scenario_source: (untyped test_case) -> untyped + def gherkin_document: (String uri) -> Cucumber::Messages::GherkinDocument? +end + +module Cucumber::Messages +end + +class Cucumber::Messages::GherkinDocument + def feature: () -> Cucumber::Messages::Feature +end + +class Cucumber::Messages::Feature + def name: () -> String end \ No newline at end of file diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs index 8a3ddac0..7ad75247 100644 --- a/vendor/rbs/rspec/0/rspec.rbs +++ b/vendor/rbs/rspec/0/rspec.rbs @@ -21,6 +21,7 @@ module RSpec::Core::ExampleGroup def run: () -> bool def top_level?: () -> bool def file_path: () -> String + def description: () -> String end end