From 1d7d3ac9d8927caf68451b936ed16e14e835bcf8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 14:09:52 +0100 Subject: [PATCH] test different ways to skip specs in rspec --- .../ci/contrib/rspec/instrumentation_spec.rb | 170 ++++++++++++++++++ spec/support/span_helpers.rb | 21 --- 2 files changed, 170 insertions(+), 21 deletions(-) diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index b7602a26..211bae93 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -208,6 +208,176 @@ def expect_failure end end + context "supports skipped examples" do + it "with skip: true" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", skip: true do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with skip: reason" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", skip: "reason in it block" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("reason in it block") + expect(first_test_span).not_to have_error + end + + it "with skip instead of it" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + skip "foo" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with xit" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + xit "foo" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Temporarily skipped with xit") + expect(first_test_span).not_to have_error + end + + it "with skip call" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + skip + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with skip call and reason given" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + skip("reason") + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("reason") + expect(first_test_span).not_to have_error + end + + it "with empty body" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Not yet implemented") + expect(first_test_span).not_to have_error + end + + it "with xcontext" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + xcontext "foo" do + it "bar" do + expect(1 + 1).to eq(5) + end + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo bar") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Temporarily skipped with xcontext") + expect(first_test_span).not_to have_error + end + + it "with pending keyword and failure" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", pending: "did not fix the math yet" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("did not fix the math yet") + expect(first_test_span).not_to have_error + end + + it "with pending keyword and passing" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", pending: "did not fix the math yet" do + expect(1 + 1).to eq(2) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::FAIL) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span).to have_error + expect(first_test_span).to have_error_message("Expected example to fail since it is pending, but it passed.") + end + + it "with pending method, reason and failure" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + pending("did not fix the math yet") + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("did not fix the math yet") + expect(first_test_span).not_to have_error + end + end + context "with git root changed" do before do expect(Datadog::CI::Utils::Git).to receive(:root).and_return("#{Dir.pwd}/spec") diff --git a/spec/support/span_helpers.rb b/spec/support/span_helpers.rb index 7bd9e5ac..27759191 100644 --- a/spec/support/span_helpers.rb +++ b/spec/support/span_helpers.rb @@ -52,27 +52,6 @@ def description_of(actual) # rubocop:disable Lint/NestedMethodDefinition define_have_error_tag(:stack, Datadog::Tracing::Metadata::Ext::Errors::TAG_STACK) define_have_error_tag(:type, Datadog::Tracing::Metadata::Ext::Errors::TAG_TYPE) - # Distributed traces have the same trace_id and parent_id as upstream parent - # span, but don't actually share the same Context with the parent. - RSpec::Matchers.define :have_distributed_parent do |parent| - match do |actual| - @matcher = have_attributes(parent_id: parent.span_id, trace_id: parent.trace_id) - @matcher.matches? actual - end - - failure_message do - @matcher.failure_message - end - end - - # Does this span have no parent span? - RSpec::Matchers.define :be_root_span do - match do |span| - value = span.parent_id - values_match? 0, value - end - end - # @param tags [Hash] key value pairs to tags/metrics to assert on RSpec::Matchers.define :have_metadata do |tags| match do |actual|