From 20bc8af477f12c0c0ba5022df889cb150458b5f9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 12:04:51 +0100 Subject: [PATCH 1/5] ci-queue support for rspec instrumentation --- lib/datadog/ci/contrib/minitest/reporter.rb | 2 -- lib/datadog/ci/contrib/rspec/example.rb | 26 +++++++++++++++++++-- lib/datadog/ci/contrib/rspec/patcher.rb | 11 ++++++++- sig/datadog/ci/contrib/rspec/example.rbs | 1 + sig/datadog/ci/contrib/rspec/patcher.rbs | 6 +++-- vendor/rbs/rspec/0/rspec.rbs | 9 +++++++ 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/lib/datadog/ci/contrib/minitest/reporter.rb b/lib/datadog/ci/contrib/minitest/reporter.rb index 2de6eb7d..bc0e1096 100644 --- a/lib/datadog/ci/contrib/minitest/reporter.rb +++ b/lib/datadog/ci/contrib/minitest/reporter.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "weakref" - require_relative "../../ext/test" require_relative "ext" diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb index 598cc860..f495a724 100644 --- a/lib/datadog/ci/contrib/rspec/example.rb +++ b/lib/datadog/ci/contrib/rspec/example.rb @@ -27,9 +27,14 @@ def run(*) test_suite_description = fetch_top_level_example_group[:description] suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}" - # remove suite name from test name to avoid duplication + # remove example group description from test name to avoid duplication test_name = test_name.sub(test_suite_description, "").strip + if ci_queue? + suite_name += " (ci-queue running example [#{test_name}])" + test_suite_span = CI.start_test_suite(suite_name) + end + CI.trace_test( test_name, suite_name, @@ -49,13 +54,25 @@ def run(*) case execution_result.status when :passed test_span.passed! + test_suite_span.passed! if test_suite_span when :failed test_span.failed!(exception: execution_result.exception) + test_suite_span.failed! if test_suite_span else - test_span.skipped!(exception: execution_result.exception) if execution_result.example_skipped? + # :pending or nil + if execution_result.pending_message + test_span.skipped!(reason: execution_result.pending_message) + elsif execution_result.example_skipped? + test_span.skipped!(exception: execution_result.exception) + else + test_span.skipped!(exception: execution_result.pending_exception) + end + test_suite_span.skipped! if test_suite_span end end + test_suite_span.finish if test_suite_span + result end end @@ -75,6 +92,11 @@ def fetch_top_level_example_group def datadog_configuration Datadog.configuration.ci[:rspec] end + + def ci_queue? + defined?(::RSpec::Queue::ExampleExtension) && + self.class.ancestors.include?(::RSpec::Queue::ExampleExtension) + end end end end diff --git a/lib/datadog/ci/contrib/rspec/patcher.rb b/lib/datadog/ci/contrib/rspec/patcher.rb index bf8ca049..7019ffbf 100644 --- a/lib/datadog/ci/contrib/rspec/patcher.rb +++ b/lib/datadog/ci/contrib/rspec/patcher.rb @@ -21,10 +21,19 @@ def target_version end def patch - ::RSpec::Core::Example.include(Example) + if ci_queue? + ::RSpec::Queue::Runner.include(Runner) + end + ::RSpec::Core::Runner.include(Runner) + ::RSpec::Core::Example.include(Example) ::RSpec::Core::ExampleGroup.include(ExampleGroup) end + + def ci_queue? + # ::RSpec::Queue::Runner is a ci-queue runner + defined?(::RSpec::Queue::Runner) + end end end end diff --git a/sig/datadog/ci/contrib/rspec/example.rbs b/sig/datadog/ci/contrib/rspec/example.rbs index 5acf73bd..fbce84bb 100644 --- a/sig/datadog/ci/contrib/rspec/example.rbs +++ b/sig/datadog/ci/contrib/rspec/example.rbs @@ -13,6 +13,7 @@ module Datadog def fetch_top_level_example_group: () -> Hash[Symbol, untyped] def datadog_configuration: () -> untyped + def ci_queue?: () -> bool end end end diff --git a/sig/datadog/ci/contrib/rspec/patcher.rbs b/sig/datadog/ci/contrib/rspec/patcher.rbs index ef9ba05d..47443508 100644 --- a/sig/datadog/ci/contrib/rspec/patcher.rbs +++ b/sig/datadog/ci/contrib/rspec/patcher.rbs @@ -5,9 +5,11 @@ module Datadog module Patcher include Datadog::Tracing::Contrib::Patcher - def self?.target_version: () -> untyped + def self?.target_version: () -> String - def self?.patch: () -> untyped + def self?.patch: () -> void + + def self?.ci_queue?: () -> bool end end end diff --git a/vendor/rbs/rspec/0/rspec.rbs b/vendor/rbs/rspec/0/rspec.rbs index 7ad75247..5ccb0357 100644 --- a/vendor/rbs/rspec/0/rspec.rbs +++ b/vendor/rbs/rspec/0/rspec.rbs @@ -4,6 +4,15 @@ end module RSpec::Core end +module RSpec::Queue +end + +module RSpec::Queue::Runner +end + +module RSpec::Queue::ExampleExtension +end + module RSpec::Core::Example def run: () -> untyped def execution_result: () -> untyped From b6a3f7e25eebd2fe64d656f8b41b43f699554997 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 12:13:24 +0100 Subject: [PATCH 2/5] add appraisals for rspec and ci-queue combination --- Appraisals | 12 ++ gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile | 31 ++++ .../jruby_9.4_ci_queue_0_rspec_3.gemfile.lock | 138 ++++++++++++++++++ gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile | 32 ++++ .../ruby_2.7_ci_queue_0_rspec_3.gemfile.lock | 137 +++++++++++++++++ gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile | 32 ++++ .../ruby_3.0_ci_queue_0_rspec_3.gemfile.lock | 137 +++++++++++++++++ gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile | 32 ++++ .../ruby_3.1_ci_queue_0_rspec_3.gemfile.lock | 137 +++++++++++++++++ gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile | 32 ++++ .../ruby_3.2_ci_queue_0_rspec_3.gemfile.lock | 137 +++++++++++++++++ gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile | 32 ++++ .../ruby_3.3_ci_queue_0_rspec_3.gemfile.lock | 137 +++++++++++++++++ 13 files changed, 1026 insertions(+) create mode 100644 gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile.lock create mode 100644 gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile.lock create mode 100644 gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile.lock create mode 100644 gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile.lock create mode 100644 gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile.lock create mode 100644 gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile create mode 100644 gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock diff --git a/Appraisals b/Appraisals index dc7ebf64..d463ab3b 100644 --- a/Appraisals +++ b/Appraisals @@ -83,10 +83,22 @@ def self.with_ci_queue_minitest_gem(minitest_versions: 5, ci_queue_versions: 0) end end +def self.with_ci_queue_rspec_gem(rspec_versions: 3, ci_queue_versions: 0) + Array(rspec_versions).each do |rspec_v| + Array(ci_queue_versions).each do |ci_queue_v| + appraise "ci-queue-#{ci_queue_v}-rspec-#{rspec_v}" do + gem "rspec", "~> #{rspec_v}" + gem "ci-queue", "~> #{ci_queue_v}" + end + end + end +end + with_minitest_gem with_rspec_gem with_cucumber_gem(versions: 3..9) with_ci_queue_minitest_gem +with_ci_queue_rspec_gem major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments diff --git a/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile b/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..6db8ba2f --- /dev/null +++ b/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,31 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..9ac678f8 --- /dev/null +++ b/gemfiles/jruby_9.4_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,138 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3-java) + json (2.7.1-java) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0) + libddwaf (1.14.0.0.0-java) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2-java) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2-java) + coderay (~> 1.1) + method_source (~> 1.0) + spoon (~> 0.0) + racc (1.7.3-java) + rainbow (3.1.1) + rake (13.1.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + spoon (0.0.6) + ffi + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + universal-java-11 + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.3.26 diff --git a/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile b/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..92f54a22 --- /dev/null +++ b/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,32 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "redcarpet" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..a262fb9b --- /dev/null +++ b/gemfiles/ruby_2.7_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3) + json (2.7.1) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0-aarch64-linux) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.3) + rainbow (3.1.1) + rake (13.1.0) + redcarpet (3.6.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + redcarpet + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.3.26 diff --git a/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile b/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..92f54a22 --- /dev/null +++ b/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,32 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "redcarpet" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..a262fb9b --- /dev/null +++ b/gemfiles/ruby_3.0_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3) + json (2.7.1) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0-aarch64-linux) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.3) + rainbow (3.1.1) + rake (13.1.0) + redcarpet (3.6.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + redcarpet + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.3.26 diff --git a/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile b/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..92f54a22 --- /dev/null +++ b/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,32 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "redcarpet" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..a262fb9b --- /dev/null +++ b/gemfiles/ruby_3.1_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3) + json (2.7.1) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0-aarch64-linux) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.3) + rainbow (3.1.1) + rake (13.1.0) + redcarpet (3.6.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + redcarpet + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.3.26 diff --git a/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile b/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..92f54a22 --- /dev/null +++ b/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,32 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "redcarpet" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..a262fb9b --- /dev/null +++ b/gemfiles/ruby_3.2_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3) + json (2.7.1) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0-aarch64-linux) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.3) + rainbow (3.1.1) + rake (13.1.0) + redcarpet (3.6.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + redcarpet + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.3.26 diff --git a/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile new file mode 100644 index 00000000..92f54a22 --- /dev/null +++ b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile @@ -0,0 +1,32 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "ddtrace" +gem "pry" +gem "rake" +gem "os" +gem "climate_control" +gem "rspec", "~> 3" +gem "rspec-collection_matchers" +gem "rspec_junit_formatter" +gem "appraisal" +gem "timecop" +gem "standard", "~> 1.31" +gem "yard" +gem "redcarpet" +gem "webrick" +gem "pimpmychangelog", ">= 0.1.2" +gem "simplecov" +gem "simplecov-cobertura", "~> 2.1.0" +gem "ci-queue", "~> 0" + +group :check do + +end + +group :development do + +end + +gemspec path: "../" diff --git a/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock new file mode 100644 index 00000000..d184c387 --- /dev/null +++ b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + datadog-ci (0.6.0) + msgpack + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + ci-queue (0.42.0) + climate_control (1.2.0) + coderay (1.1.3) + ddtrace (1.19.0) + datadog-ci (~> 0.6.0) + debase-ruby_core_source (= 3.3.1) + libdatadog (~> 5.0.0.1.0) + libddwaf (~> 1.14.0.0.0) + msgpack + debase-ruby_core_source (3.3.1) + diff-lcs (1.5.0) + docile (1.4.0) + ffi (1.16.3) + json (2.7.1) + language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0-aarch64-linux) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) + lint_roller (1.1.0) + method_source (1.0.0) + msgpack (1.7.2) + os (1.1.4) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pimpmychangelog (0.1.3) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.3) + rainbow (3.1.1) + rake (13.1.0) + redcarpet (3.6.0) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rspec_junit_formatter (0.6.0) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.20.2) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-cobertura (2.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + standard (1.33.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.59.0) + standard-custom (~> 1.0.0) + standard-performance (~> 1.3) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.3.1) + lint_roller (~> 1.1) + rubocop-performance (~> 1.20.2) + thor (1.3.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + webrick (1.8.1) + yard (0.9.34) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + appraisal + ci-queue (~> 0) + climate_control + datadog-ci! + ddtrace + os + pimpmychangelog (>= 0.1.2) + pry + rake + redcarpet + rspec (~> 3) + rspec-collection_matchers + rspec_junit_formatter + simplecov + simplecov-cobertura (~> 2.1.0) + standard (~> 1.31) + timecop + webrick + yard + +BUNDLED WITH + 2.4.19 From 1e2ade0e5d8ab23ff1906426e723eaec3c9dd6fe Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 12:17:24 +0100 Subject: [PATCH 3/5] add ci_queue_rspec specs to the Rakefile --- Rakefile | 6 +++++- .../ruby_3.3_ci_queue_0_rspec_3.gemfile.lock | 4 ++++ .../ci_queue_minitest/instrumentation_spec.rb | 1 - .../ci_queue_rspec/instrumentation_spec.rb | 21 +++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb diff --git a/Rakefile b/Rakefile index 6a62924f..2e9733b0 100644 --- a/Rakefile +++ b/Rakefile @@ -42,6 +42,9 @@ TEST_METADATA = { }, "ci_queue_minitest" => { "ci-queue-0-minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby" + }, + "ci_queue_rspec" => { + "ci-queue-0-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby" } } @@ -105,7 +108,8 @@ namespace :spec do :cucumber, :rspec, :minitest, - :ci_queue_minitest + :ci_queue_minitest, + :ci_queue_rspec ].each do |contrib| desc "" # "Explicitly hiding from `rake -T`" RSpec::Core::RakeTask.new(contrib) do |t, args| diff --git a/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock index d184c387..5ce118dc 100644 --- a/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock +++ b/gemfiles/ruby_3.3_ci_queue_0_rspec_3.gemfile.lock @@ -27,9 +27,12 @@ GEM ffi (1.16.3) json (2.7.1) language_server-protocol (3.17.0.3) + libdatadog (5.0.0.1.0) libdatadog (5.0.0.1.0-aarch64-linux) libddwaf (1.14.0.0.0-aarch64-linux) ffi (~> 1.0) + libddwaf (1.14.0.0.0-arm64-darwin) + ffi (~> 1.0) lint_roller (1.1.0) method_source (1.0.0) msgpack (1.7.2) @@ -111,6 +114,7 @@ GEM PLATFORMS aarch64-linux + arm64-darwin-23 DEPENDENCIES appraisal diff --git a/spec/datadog/ci/contrib/ci_queue_minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/ci_queue_minitest/instrumentation_spec.rb index b1542155..110a1b71 100644 --- a/spec/datadog/ci/contrib/ci_queue_minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/ci_queue_minitest/instrumentation_spec.rb @@ -8,7 +8,6 @@ end let(:run_id) { rand(1..2**64 - 1) } - let(:queue_file_path) { "#{Dir.pwd}/tmp/ci-queue-#{run_id}" } before do Minitest::Runnable.reset diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb new file mode 100644 index 00000000..37b07adc --- /dev/null +++ b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb @@ -0,0 +1,21 @@ +require "fileutils" + +RSpec.describe "RSpec instrumentation with Shopify's ci-queue runner" do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + let(:run_id) { rand(1..2**64 - 1) } + + before do + FileUtils.mkdir("log") + end + + after do + FileUtils.rm_rf("log") + end + + it "instruments this rspec session" do + expect(1 + 1).to eq(2) + end +end From 930662b7e77c05162a3073293123d84bfd8f69bd Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 13:04:25 +0100 Subject: [PATCH 4/5] integration specs for ci-queue and rspec combination --- .../ci_queue_rspec/instrumentation_spec.rb | 74 ++++++++++++++++++- .../suite_under_test/some_test_rspec.rb | 13 ++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb index 37b07adc..fef844b7 100644 --- a/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/ci_queue_rspec/instrumentation_spec.rb @@ -1,3 +1,4 @@ +require "rspec/queue" require "fileutils" RSpec.describe "RSpec instrumentation with Shopify's ci-queue runner" do @@ -6,6 +7,21 @@ end let(:run_id) { rand(1..2**64 - 1) } + let(:options) do + RSpec::Core::ConfigurationOptions.new([ + "-Ispec/datadog/ci/contrib/ci_queue_rspec/suite_under_test", + "--queue", + "list:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A1%5D:.%2Fspec%2Fdatadog%2Fci%2Fcontrib%2Fci_queue_rspec%2Fsuite_under_test%2Fsome_test_rspec.rb%5B1%3A1%3A2%5D", + "--require", + "some_test_rspec.rb", + "--build", + run_id.to_s, + "--worker", + "1", + "--default-path", + "spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test" + ]) + end before do FileUtils.mkdir("log") @@ -15,7 +31,63 @@ FileUtils.rm_rf("log") end + def devnull + File.new("/dev/null", "w") + end + + # Yields to a block in a new RSpec global context. All RSpec + # test configuration and execution should be wrapped in this method. + def with_new_rspec_environment + old_configuration = ::RSpec.configuration + old_world = ::RSpec.world + ::RSpec.configuration = ::RSpec::Core::Configuration.new + ::RSpec.world = ::RSpec::Core::World.new + + yield + ensure + ::RSpec.configuration = old_configuration + ::RSpec.world = old_world + end + it "instruments this rspec session" do - expect(1 + 1).to eq(2) + with_new_rspec_environment do + ::RSpec::Queue::Runner.new(options).run(devnull, devnull) + end + + # test session and module traced + expect(test_session_span).not_to be_nil + expect(test_module_span).not_to be_nil + + # test session and module are failed + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + + # test suite spans are created for each test as for parallel execution + expect(test_suite_spans).to have(2).items + expect(test_suite_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS) }.sort).to eq( + [Datadog::CI::Ext::Test::Status::FAIL, Datadog::CI::Ext::Test::Status::PASS] + ) + expect(test_suite_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort).to eq( + [ + "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested fails])", + "SomeTest at ./spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb (ci-queue running example [nested foo])" + ] + ) + + # there is test span for every test case + expect(test_spans).to have(2).items + # each test span has its own test suite + expect(test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq).to have(2).items + + # every test span is connected to test module and test session + test_spans.each do |test_span| + [Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID, Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID].each do |tag| + expect(test_span.get_tag(tag)).not_to be_nil + end + end end end diff --git a/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb b/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb new file mode 100644 index 00000000..7b775cb9 --- /dev/null +++ b/spec/datadog/ci/contrib/ci_queue_rspec/suite_under_test/some_test_rspec.rb @@ -0,0 +1,13 @@ +require "rspec" + +RSpec.describe "SomeTest" do + context "nested" do + it "foo" do + # DO NOTHING + end + + it "fails" do + expect(1).to eq(2) + end + end +end From 1d7d3ac9d8927caf68451b936ed16e14e835bcf8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 23 Jan 2024 14:09:52 +0100 Subject: [PATCH 5/5] test different ways to skip specs in rspec --- .../ci/contrib/rspec/instrumentation_spec.rb | 170 ++++++++++++++++++ spec/support/span_helpers.rb | 21 --- 2 files changed, 170 insertions(+), 21 deletions(-) diff --git a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb index b7602a26..211bae93 100644 --- a/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/rspec/instrumentation_spec.rb @@ -208,6 +208,176 @@ def expect_failure end end + context "supports skipped examples" do + it "with skip: true" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", skip: true do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with skip: reason" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", skip: "reason in it block" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("reason in it block") + expect(first_test_span).not_to have_error + end + + it "with skip instead of it" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + skip "foo" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with xit" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + xit "foo" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Temporarily skipped with xit") + expect(first_test_span).not_to have_error + end + + it "with skip call" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + skip + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("No reason given") + expect(first_test_span).not_to have_error + end + + it "with skip call and reason given" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + skip("reason") + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("reason") + expect(first_test_span).not_to have_error + end + + it "with empty body" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Not yet implemented") + expect(first_test_span).not_to have_error + end + + it "with xcontext" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + xcontext "foo" do + it "bar" do + expect(1 + 1).to eq(5) + end + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo bar") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("Temporarily skipped with xcontext") + expect(first_test_span).not_to have_error + end + + it "with pending keyword and failure" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", pending: "did not fix the math yet" do + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("did not fix the math yet") + expect(first_test_span).not_to have_error + end + + it "with pending keyword and passing" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo", pending: "did not fix the math yet" do + expect(1 + 1).to eq(2) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::FAIL) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span).to have_error + expect(first_test_span).to have_error_message("Expected example to fail since it is pending, but it passed.") + end + + it "with pending method, reason and failure" do + with_new_rspec_environment do + RSpec.describe "some skipped test" do + it "foo" do + pending("did not fix the math yet") + expect(1 + 1).to eq(5) + end + end.run + end + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::SKIP) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SKIP_REASON)).to eq("did not fix the math yet") + expect(first_test_span).not_to have_error + end + end + context "with git root changed" do before do expect(Datadog::CI::Utils::Git).to receive(:root).and_return("#{Dir.pwd}/spec") diff --git a/spec/support/span_helpers.rb b/spec/support/span_helpers.rb index 7bd9e5ac..27759191 100644 --- a/spec/support/span_helpers.rb +++ b/spec/support/span_helpers.rb @@ -52,27 +52,6 @@ def description_of(actual) # rubocop:disable Lint/NestedMethodDefinition define_have_error_tag(:stack, Datadog::Tracing::Metadata::Ext::Errors::TAG_STACK) define_have_error_tag(:type, Datadog::Tracing::Metadata::Ext::Errors::TAG_TYPE) - # Distributed traces have the same trace_id and parent_id as upstream parent - # span, but don't actually share the same Context with the parent. - RSpec::Matchers.define :have_distributed_parent do |parent| - match do |actual| - @matcher = have_attributes(parent_id: parent.span_id, trace_id: parent.trace_id) - @matcher.matches? actual - end - - failure_message do - @matcher.failure_message - end - end - - # Does this span have no parent span? - RSpec::Matchers.define :be_root_span do - match do |span| - value = span.parent_id - values_match? 0, value - end - end - # @param tags [Hash] key value pairs to tags/metrics to assert on RSpec::Matchers.define :have_metadata do |tags| match do |actual|